Adds a "global" farmer to the standard collection

adaptation/covariates.py

@@ -32,7 +32,7 @@
 from openest.generate import fast_dataset
 from impactlab_tools.utils import files
 from .econmodel import *
-from datastore import agecohorts, irvalues, irregions
+from datastore import agecohorts, irvalues, irregions, population
 from climate.yearlyreader import RandomYearlyAccess
 from interpret import averages, configs
 
@@ -1370,3 +1370,60 @@ def get_current(self, region):
         """
         covars = self.source.get_current(region)
         return self.make_power(covars)
+
+class GlobalAggregatedCovariator(Covariator):
+    """Spatially average the covariates across all regions.
+
+    Parameters
+    ----------
+    source
+    maxbaseline
+    """
+    def __init__(self, source, maxbaseline, config=None):
+        super(GlobalAggregatedCovariator, self).__init__(maxbaseline, config=config)
+        pop_baseline_withyear = population.population_baseline_data(2000, maxbaseline, [], add_adm0=False)
+        self.regions = pop_baseline_withyear.keys()
+        self.pop_baseline = [np.mean(list(pop_baseline_withyear[region].values())) for region in self.regions]
+        self.source = source
+        self.year_of_cache = None
+        self.byregion_cache = None # {region: { key-local: value }}
+        self.global_cache = None # {key: value }
+
+    def get_current(self, region):
+        """
+        Parameters
+        ----------
+        country : str
+
+        Returns
+        -------
+        dict
+        """
+        if self.byregion_cache:
+            return {**{(key + '-local'): self.byregion_cache[region][key] for key in self.byregion_cache[region]}, **self.global_cache}
+
+        self.byregion_cache = {region: self.source.get_current(region) for region in self.regions}
+
+        self.global_cache = {}
+        for key in self.byregion_cache[list(self.regions)[0]]:
+            regionvalues = [self.byregion_cache[region][key] for region in self.regions]
+            self.global_cache[key] = np.average(regionvalues, weights=self.pop_baseline)
+
+        return self.get_current(region)
+
+    def get_update(self, region, year, ds):
+        """
+        Parameters
+        ----------
+        region : str
+        year : int
+        ds : xarray.Dataset
+        """
+        self.source.get_update(region, year, ds)
+
+        if self.year_of_cache == year:
+            return # already done
+
+        self.year_of_cache = year
+        self.byregion_cache = None
+        self.global_cache = None

adaptation/curvegen.py

@@ -34,6 +34,7 @@
 from openest.generate.curvegen import *
 from openest.generate import checks, fast_dataset, formatting, smart_curve, formattools
 from openest.models.curve import FlatCurve
+from .covariates import GlobalAggregatedCovariator
 
 region_curves = {}
 
@@ -195,7 +196,10 @@ class FarmerCurveGenerator(DelayedCurveGenerator):
     """
     def __init__(self, curvegen, covariator, farmer='full', save_curve=True, endbaseline=2015):
         super(FarmerCurveGenerator, self).__init__(curvegen)
-        self.covariator = covariator
+        if farmer == 'global':
+            self.covariator = GlobalAggregatedCovariator(covariator, endbaseline)
+        else:
+            self.covariator = covariator
         self.farmer = farmer
         self.save_curve = save_curve
         self.lincom_last_covariates = {}
@@ -236,7 +240,7 @@ def get_next_curve(self, region, year, *args, **kwargs):
         if self.farmer == 'full':
             covariates = self.covariator.offer_update(region, year, kwargs['weather'])
             curve = self.curvegen.get_curve(region, year, covariates)
-        elif self.farmer == 'noadapt':
+        elif self.farmer in ['noadapt', 'global']:
             curve = self.last_curves[region]
         elif self.farmer == 'incadapt':
             covariates = self.covariator.offer_update(region, year, None)

datastore/population.py

@@ -41,9 +41,9 @@ def each_future_population(model, scenario, dependencies):
 
             yield region, year, value
 
-def population_baseline_data(year0, year1, dependencies):
+def population_baseline_data(year0, year1, dependencies, add_adm0=True):
     global population_baseline_cache
-    if (year0, year1) in population_baseline_cache:
+    if add_adm0 and (year0, year1) in population_baseline_cache:
         return population_baseline_cache[year0, year1]
 
     baselinedata = {} # {region: {year: value}}
@@ -66,14 +66,18 @@ def population_baseline_data(year0, year1, dependencies):
 
             if len(region) > 3 and region[3] == '.':
                 if region[:3] not in adm0s:
-                    adm0s[region[:3]] = 0
-                adm0s[region[:3]] += value
-
-        for adm0 in adm0s:
-            assert adm0 not in baselinedata
-            baselinedata[adm0] = adm0s[adm0]
-
-    population_baseline_cache[year0, year1] = baselinedata
+                    adm0s[region[:3]] = {}
+                if year not in adm0s[region[:3]]:
+                    adm0s[region[:3]][year] = 0
+                adm0s[region[:3]][year] += value
+
+        if add_adm0:
+            for adm0 in adm0s:
+                assert adm0 not in baselinedata
+                baselinedata[adm0] = adm0s[adm0]
+
+    if add_adm0: # Only save if we have ADM0 as well
+        population_baseline_cache[year0, year1] = baselinedata
     return baselinedata
 
 def extend_population_future(baselinedata, year0, year1, regions, model, scenario, dependencies):

docs/generate.md

@@ -34,10 +34,13 @@ included in the generate configuration file.
    a median or monte carlo run, but setting this to `false` will skip
    them.
 
- - `do_farmers`: true, false, or 'always'; if true, alternative
-   assumptions of adaptation (income-only and no-adaptation) will be
-   generated.  If 'always', alternative adaptation assumptions will be
-   calculated even with historical climate.
+ - `do_farmers`: true, false, 'always', or a list of 'noadapt',
+   'incadapt', 'global', 'histclim-noadapt', 'histclim-incadapt', and
+   'histclim-global'; if true, alternative assumptions of adaptation
+   (income-only and no-adaptation) will be generated.  If 'always',
+   alternative adaptation assumptions will be calculated even with
+   historical climate. To generate arbitrary adaptation assumptions,
+   provide a list, e.g., ['noadapt', 'incadapt', 'global'].
 
  - `do_single`: true or false (default): Should we stop after a single
    target directory?

generate/loadmodels.py

@@ -16,10 +16,14 @@ def single(bundle_iterator):
     for econ_model, econ_scenario, economicmodel in covariates.iterate_econmodels():
         allecons.append((econ_scenario, econ_model, economicmodel))
 
+    assert allecons, "Cannot find economic models."
+
     allclims = []
     for clim_scenario, clim_model, weatherbundle in bundle_iterator:
         allclims.append((clim_scenario, clim_model, weatherbundle))
 
+    assert allclims, "Cannot find climate models."
+
     allexogenous = []
     for econ_scenario, econ_model, economicmodel in allecons:
         for clim_scenario, clim_model, weatherbundle in allclims:
@@ -35,6 +39,8 @@ def single(bundle_iterator):
 
             return clim_scenario, clim_model, weatherbundle, econ_scenario, econ_model, economicmodel
 
+    assert False, "Could not find an appropriate single model."
+
 def random_order(bundle_iterator, config=None):
     if config is None:
         config = {}

interpret/calcspec.py

@@ -31,7 +31,7 @@ def prepare_interp_raw(csvv, weatherbundle, economicmodel, qvals, farmer="full",
     weatherbundle : generate.weather.WeatherBundle
     economicmodel : adaptation.econmodel.SSPEconomicModel
     qvals : generate.pvalses.ConstantDictionary
-    farmer : {"full", "noadapt", "incadapt"}, optional
+    farmer : {"full", "noadapt", "incadapt", "global"}, optional
         Adaptation scheme to use.
     specconf : dict or None, optional
         This is the model containing 'specifications' and 'calculation' keys.

interpret/configs.py

@@ -472,7 +472,10 @@ def items(self):
         copydict = dict(self.parent.items())
         copydict.update(dict(self.child.items()))
         return copydict.items()
-
+
+    def check_usage(self):
+        return self.parent.check_usage() + self.child.check_usage()
+
 class ConfigList(MutableSequence):
     """Wrapper on lists contained in configuration dictionaries to monitor key access.
 

interpret/container.py

@@ -38,6 +38,17 @@ def check_doit(targetdir, basename, suffix, config, deletebad=False):
 
     return False
 
+def check_dofarmer(farmer, config, weatherbundle):
+    farmers = config.get('do_farmers', [])
+    if farmers == False:
+        farmers = []
+    elif farmers == True:
+        farmers = ['noadapt', 'incadapt']
+    elif farmers == 'always':
+        farmers = ['noadapt', 'incadapt', 'histclim-noadapt', 'histclim-incadapt']
+
+    return farmer in farmers or (weatherbundle.is_historical() and ('histclim-' + farmer) in farmers)
+
 def get_modules(config):
     models = config['models']
     for model in models:
@@ -125,17 +136,17 @@ def produce_csvv(basename, csvv, module, specconf, targetdir, weatherbundle, eco
 
     if profile:
         return
-        
-    if config.get('do_farmers', False) and (not weatherbundle.is_historical() or config['do_farmers'] == 'always'):
-        # Lock in the values
-        pvals[basename].lock()
-
-        if check_doit(targetdir, basename + "-noadapt", suffix, config):
-            print("No adaptation")
-            calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer='noadapt', config=config, standard=False)
-            effectset.generate(targetdir, basename + "-noadapt" + suffix, weatherbundle, calculation, specconf['description'] + ", with no adaptation.", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv)
-
-        if check_doit(targetdir, basename + "-incadapt", suffix, config):
-            print("Income-only adaptation")
-            calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer='incadapt', config=config, standard=False)
-            effectset.generate(targetdir, basename + "-incadapt" + suffix, weatherbundle, calculation, specconf['description'] + ", with interpolation and only environmental adaptation.", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv)
+
+    # Do farmers, if requested
+    suffixes = {'noadapt': "with no adaptation",
+                'incadapt': "with interpolation and only environmental adaptation",
+                'global': "with no adaptation and global covariates"}
+
+    for farmer, explain in suffixes.items():
+        if check_dofarmer(farmer, config, weatherbundle) and check_doit(targetdir, basename + "-" + farmer, suffix, config):
+            # Lock in the values
+            pvals[basename].lock()
+
+            print("Limited adaptation: " + explain)
+            calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer=farmer, config=config, standard=False)
+            effectset.generate(targetdir, basename + "-" + farmer + suffix, weatherbundle, calculation, specconf['description'] + ", " + explain + ".", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv)

interpret/specification.py

@@ -8,7 +8,7 @@
 precipitation.
 """
 
-import re
+import re, copy
 from collections.abc import Mapping, Sequence
 from adaptation import csvvfile, curvegen, curvegen_known, curvegen_arbitrary, covariates, constraints, parallel_covariates, parallel_econmodel
 from generate import parallel_weather
@@ -123,7 +123,7 @@ def create_covariator(specconf, weatherbundle, economicmodel, config=None, quiet
 
     Parameters
     ----------
-    specconf : dict, optional
+    specconf : dict
         Specification configuration.
     weatherbundle : generate.weather.DailyWeatherBundle
     economicmodel : adaptation.econmodel.SSPEconomicModel
@@ -136,6 +136,16 @@ def create_covariator(specconf, weatherbundle, economicmodel, config=None, quiet
     """
     if config is None:
         config = {}
+
+    if farmer == 'global':
+        # Need all regions in covariates for global
+        if 'filter-region' in config:
+            config = configs.shallow_copy(config)
+            del config['filter-region']
+        if 'filter-region' in specconf:
+            specconf = configs.shallow_copy(specconf)
+            del specconf['filter-region']
+
     if parallel_weather.is_parallel(weatherbundle) and parallel_econmodel.is_parallel(economicmodel):
         return parallel_covariates.create_covariator(specconf, weatherbundle, economicmodel, farmer)
     if 'covariates' in specconf:
@@ -164,7 +174,7 @@ def create_curvegen(csvv, covariator, regions, farmer='full', specconf=None, get
         Various parameters and curve descriptions from CSVV file.
     covariator : adaptation.covariates.Covariator or None
     regions : xarray.Dataset
-    farmer : {'full', 'noadapt', 'incadapt'}, optional
+    farmer : {'full', 'noadapt', 'incadapt', 'global'}, optional
         Type of farmer adaptation.
     specconf : dict, optional
         Specification configuration.
@@ -475,7 +485,7 @@ def prepare_interp_raw(csvv, weatherbundle, economicmodel, qvals, farmer='full',
     weatherbundle : generate.weather.DailyWeatherBundle
     economicmodel : adaptation.econmodel.SSPEconomicModel
     qvals : generate.pvalses.ConstantDictionary
-    farmer : {'full', 'noadapt', 'incadapt'}, optional
+    farmer : {'full', 'noadapt', 'incadapt', 'global'}, optional
         Type of farmer adaptation.
     specconf : dict, optional
         Specification configuration.
@@ -506,7 +516,7 @@ def prepare_interp_raw(csvv, weatherbundle, economicmodel, qvals, farmer='full',
         )
 
     depenunit = specconf['depenunit']
-    
+
     covariator = create_covariator(specconf, weatherbundle, economicmodel, config, farmer=farmer)
 
     # Subset to regions (i.e. hierids) to act on.