Merge pull request #670 from Breakthrough-Energy/develop

develop -> master

.gitignore

@@ -1,6 +1,7 @@
 # This is specific to this package
 powersimdata/utility/.server_user
 config.ini
+powersimdata/network/europe_tub/data/*
 
 # The remainder of this file taken from github/gitignore
 # https://github.com/github/gitignore/blob/master/Python.gitignore

Pipfile

@@ -9,6 +9,7 @@ pytest = "*"
 coverage = "*"
 pytest-cov = "*"
 pypsa = "*"
+zenodo_get = "*"
 
 [packages]
 networkx = "~=2.5"
@@ -20,4 +21,4 @@ tqdm = "==4.29.1"
 requests = "~=2.25"
 fs = "==2.4.14"
 "fs.sshfs" = "*"
-fs-azureblob = "*"
+fs-azureblob = ">=0.2.1"

powersimdata/data_access/context.py

@@ -8,18 +8,20 @@ class Context:
     """Factory for data access instances"""
 
     @staticmethod
-    def get_data_access():
+    def get_data_access(make_fs=None):
         """Return a data access instance appropriate for the current
         environment.
 
+        :param callable make_fs: a function that returns a filesystem instance, or
+            None to use a default
         :return: (:class:`powersimdata.data_access.data_access.DataAccess`) -- a data access
             instance
         """
-        root = server_setup.DATA_ROOT_DIR
-
         if server_setup.DEPLOYMENT_MODE == DeploymentMode.Server:
-            return SSHDataAccess(root)
-        return LocalDataAccess(root)
+            if make_fs is None:
+                make_fs = lambda: None  # noqa: E731
+            return SSHDataAccess(make_fs())
+        return LocalDataAccess()
 
     @staticmethod
     def get_launcher(scenario):

powersimdata/data_access/data_access.py

@@ -16,9 +16,8 @@
 class DataAccess:
     """Interface to a local or remote data store."""
 
-    def __init__(self, root):
+    def __init__(self):
         """Constructor"""
-        self.root = root
         self.join = fs.path.join
         self.local_fs = None
 
@@ -163,10 +162,10 @@ def push(self, file_name, checksum):
 class LocalDataAccess(DataAccess):
     """Interface to shared data volume"""
 
-    def __init__(self, root=server_setup.LOCAL_DIR):
-        super().__init__(root)
-        self.local_fs = fs.open_fs(root)
-        self.fs = self._get_fs()
+    def __init__(self, _fs=None):
+        super().__init__()
+        self.local_fs = fs.open_fs(server_setup.LOCAL_DIR)
+        self.fs = _fs if _fs is not None else self._get_fs()
 
     def _get_fs(self):
         mfs = MultiFS()
@@ -193,18 +192,19 @@ def push(self, file_name, checksum):
 class SSHDataAccess(DataAccess):
     """Interface to a remote data store, accessed via SSH."""
 
-    def __init__(self, root=server_setup.DATA_ROOT_DIR):
+    def __init__(self, _fs=None):
         """Constructor"""
-        super().__init__(root)
-        self._fs = None
+        super().__init__()
+        self.root = server_setup.DATA_ROOT_DIR
+        self._fs = _fs
         self.local_fs = fs.open_fs(server_setup.LOCAL_DIR)
 
     @property
     def fs(self):
-        """Get or create the filesystem object
+        """Get or create a filesystem object, defaulting to a MultiFS that combines the
+        server and blob containers.
 
-        :raises IOError: if connection failed or still within retry window
-        :return: (*fs.multifs.MultiFS*) -- filesystem instance
+        :return: (*fs.base.FS*) -- filesystem instance
         """
         if self._fs is None:
             self._fs = get_multi_fs(self.root)

powersimdata/data_access/fs_helper.py

@@ -11,8 +11,9 @@ def get_blob_fs(container):
     :param str container: the container name
     :return: (*fs.base.FS*) -- filesystem instance
     """
-    account = "besciences"
-    return fs.open_fs(f"azblob://{account}@{container}")
+    account = "esmi"
+    sas_token = server_setup.BLOB_TOKEN_RO
+    return fs.open_fs(f"azblobv2://{account}:{sas_token}@{container}")
 
 
 def get_ssh_fs(root=""):
@@ -49,3 +50,22 @@ def get_multi_fs(root):
     remotes = ",".join([f[0] for f in mfs.iterate_fs()])
     print(f"Initialized remote filesystem with {remotes}")
     return mfs
+
+
+def get_scenario_fs():
+    """Create filesystem combining the server (if connected) with blob storage,
+    prioritizing the server if connected.
+
+    :return: (*fs.base.FS*) -- filesystem instance
+    """
+    scenario_data = get_blob_fs("scenariodata")
+    mfs = MultiFS()
+    try:
+        ssh_fs = get_ssh_fs(server_setup.DATA_ROOT_DIR)
+        mfs.add_fs("ssh_fs", ssh_fs, write=True, priority=2)
+    except:  # noqa
+        print("Could not connect to ssh server")
+    mfs.add_fs("scenario_fs", scenario_data, priority=1)
+    remotes = ",".join([f[0] for f in mfs.iterate_fs()])
+    print(f"Initialized remote filesystem with {remotes}")
+    return mfs

powersimdata/design/investment/investment_costs.py

@@ -291,9 +291,7 @@ def calculate_dc_inv_costs(scenario, sum_results=True, base_grid=None):
         _check_grid_models_match(base_grid, grid_differences)
 
     # find upgraded DC lines
-    capacity_difference = calculate_dcline_difference(
-        base_grid.dcline, grid_differences.dcline
-    )
+    capacity_difference = calculate_dcline_difference(base_grid, grid_differences)
     grid_differences.dcline = grid_differences.dcline.assign(
         Pmax=capacity_difference["diff"].to_numpy()
     )

powersimdata/design/transmission/statelines.py

@@ -1,6 +1,3 @@
-from powersimdata.network.usa_tamu.constants.zones import id2abv
-
-
 def classify_interstate_intrastate(scenario):
     """Classifies branches in a change_table as interstate or intrastate.
 
@@ -27,6 +24,7 @@ def _classify_interstate_intrastate(ct, grid):
     """
 
     branch = grid.branch
+    id2abv = grid.model_immutables.zones["id2abv"]
     upgraded_branches = {"interstate": [], "intrastate": []}
 
     if "branch" not in ct or "branch_id" not in ct["branch"]:

powersimdata/input/abstract_grid.py

@@ -20,6 +20,8 @@ def __init__(self):
         self.bus = pd.DataFrame()
         self.branch = pd.DataFrame()
         self.storage = storage_template()
+        self.grid_model = ""
+        self.model_immutables = None
 
 
 def storage_template():
@@ -38,7 +40,7 @@ def storage_template():
         "InEff": None,
         "OutEff": None,
         "LossFactor": None,  # stored energy fraction / hour
-        "energy_price": None,  # $/MWh
+        "energy_value": None,  # $/MWh
         "terminal_min": None,
         "terminal_max": None,
     }

powersimdata/input/check.py

@@ -211,7 +211,7 @@ def _check_connected_components(grid, error_messages):
     num_connected_components = len([c for c in nx.connected_components(g)])
     if len(grid.interconnect) == 1:
         # Check for e.g. ['USA'] interconnect, which is really three interconnects
-        interconnect_aliases = grid.model_immutables.zones["interconnect_combinations"]
+        interconnect_aliases = grid.model_immutables.zones["name2interconnect"]
         if grid.interconnect[0] in interconnect_aliases:
             num_interconnects = len(interconnect_aliases[grid.interconnect[0]])
         else:
@@ -299,19 +299,18 @@ def _check_grid_type(grid):
         raise TypeError(f"grid must be a {_grid.Grid} object")
 
 
-def _check_areas_and_format(areas, grid_model="usa_tamu"):
+def _check_areas_and_format(areas, mi=ModelImmutables("usa_tamu")):
     """Ensure that areas are valid. Duplicates are removed and state abbreviations are
     converted to their actual name.
 
     :param str/list/tuple/set areas: areas(s) to check. Could be load zone name(s),
         state name(s)/abbreviation(s) or interconnect(s).
-    :param str grid_model: grid model.
-    :raises TypeError: if areas is not a list/tuple/set of str.
-    :raises ValueError: if areas is empty or not valid.
-    :return: (*set*) -- areas as a set. State abbreviations are converted to state
-        names.
+    :param powersimdata.network.model.ModelImmutables mi: immutables of a grid model.
+    :raises TypeError: if ``areas`` is not a list/tuple/set of str.
+    :raises ValueError: if ``areas`` is empty or not valid.
+    :return: (*set*) -- areas as a set. State/Country abbreviations are converted to
+        state/country names.
     """
-    mi = ModelImmutables(grid_model)
     if isinstance(areas, str):
         areas = {areas}
     elif isinstance(areas, (list, set, tuple)):
@@ -322,34 +321,28 @@ def _check_areas_and_format(areas, grid_model="usa_tamu"):
         raise TypeError("areas must be a str or a list/tuple/set of str")
     if len(areas) == 0:
         raise ValueError("areas must be non-empty")
-    all_areas = (
-        mi.zones["loadzone"]
-        | mi.zones["abv"]
-        | mi.zones["state"]
-        | mi.zones["interconnect"]
-    )
+    all_areas = set().union(*(mi.zones[z] for z in mi.zones["mappings"]))
     if not areas <= all_areas:
         diff = areas - all_areas
         raise ValueError("invalid area(s): %s" % " | ".join(diff))
 
     abv_in_areas = [z for z in areas if z in mi.zones["abv"]]
     for a in abv_in_areas:
         areas.remove(a)
-        areas.add(mi.zones["abv2state"][a])
+        areas.add(mi.zones[f"abv2{mi.zones['division']}"][a])
 
     return areas
 
 
-def _check_resources_and_format(resources, grid_model="usa_tamu"):
+def _check_resources_and_format(resources, mi=ModelImmutables("usa_tamu")):
     """Ensure that resources are valid and convert variable to a set.
 
     :param str/list/tuple/set resources: resource(s) to check.
-    :param str grid_model: grid model.
+    :param powersimdata.network.model.ModelImmutables mi: immutables of a grid model.
     :raises TypeError: if resources is not a list/tuple/set of str.
     :raises ValueError: if resources is empty or not valid.
     :return: (*set*) -- resources as a set.
     """
-    mi = ModelImmutables(grid_model)
     if isinstance(resources, str):
         resources = {resources}
     elif isinstance(resources, (list, set, tuple)):
@@ -366,17 +359,17 @@ def _check_resources_and_format(resources, grid_model="usa_tamu"):
     return resources
 
 
-def _check_resources_are_renewable_and_format(resources, grid_model="usa_tamu"):
+def _check_resources_are_renewable_and_format(
+    resources, mi=ModelImmutables("usa_tamu")
+):
     """Ensure that resources are valid renewable resources and convert variable to
     a set.
-
+    :param powersimdata.network.model.ModelImmutables mi: immutables of a grid model.
     :param str/list/tuple/set resources: resource(s) to analyze.
-    :param str grid_model: grid model.
     :raises ValueError: if resources are not renewables.
     return: (*set*) -- resources as a set
     """
-    mi = ModelImmutables(grid_model)
-    resources = _check_resources_and_format(resources, grid_model=grid_model)
+    resources = _check_resources_and_format(resources, mi=mi)
     if not resources <= mi.plants["renewable_resources"]:
         diff = resources - mi.plants["all_resources"]
         raise ValueError("invalid renewable resource(s): %s" % " | ".join(diff))

powersimdata/input/converter/csv_to_grid.py

@@ -0,0 +1,71 @@
+import os
+
+from powersimdata.input.abstract_grid import AbstractGrid
+from powersimdata.input.converter.helpers import (
+    add_coord_to_grid_data_frames,
+    add_zone_to_grid_data_frames,
+)
+from powersimdata.network.constants.model import model2region
+from powersimdata.network.csv_reader import CSVReader
+
+
+class FromCSV(AbstractGrid):
+    """Grid Builder for grid model enclosed in CSV files."""
+
+    def _set_data_loc(self, top_dirname):
+        """Sets data location.
+
+        :param str top_dirname: name of directory enclosing data.
+        :raises IOError: if directory does not exist.
+        """
+        data_loc = os.path.join(top_dirname, "data")
+        if os.path.isdir(data_loc) is False:
+            raise IOError("%s directory not found" % data_loc)
+        else:
+            self.data_loc = data_loc
+
+    def _build(self, interconnect, grid_model):
+        """Build network.
+
+        :param list interconnect: interconnect name(s).
+        :param str model: the grid model.
+        """
+        reader = CSVReader(self.data_loc)
+        self.bus = reader.bus
+        self.plant = reader.plant
+        self.branch = reader.branch
+        self.dcline = reader.dcline
+        self.gencost["after"] = self.gencost["before"] = reader.gencost
+        self.sub = reader.sub
+        self.bus2sub = reader.bus2sub
+        self.id2zone = reader.zone["zone_name"].to_dict()
+        self.zone2id = {v: k for k, v in self.id2zone.items()}
+
+        self._add_information()
+
+        if model2region[grid_model] not in interconnect:
+            self._drop_interconnect(interconnect)
+
+    def _add_information(self):
+        add_zone_to_grid_data_frames(self)
+        add_coord_to_grid_data_frames(self)
+
+    def _drop_interconnect(self, interconnect):
+        """Trim data frames to only keep information pertaining to the user
+        defined interconnect(s).
+
+        :param list interconnect: interconnect name(s).
+        """
+        for key, value in self.__dict__.items():
+            if key in ["sub", "bus2sub", "bus", "plant", "branch"]:
+                value.query("interconnect == @interconnect", inplace=True)
+            elif key == "gencost":
+                value["before"].query("interconnect == @interconnect", inplace=True)
+            elif key == "dcline":
+                value.query(
+                    "from_interconnect == @interconnect &"
+                    "to_interconnect == @interconnect",
+                    inplace=True,
+                )
+        self.id2zone = {k: self.id2zone[k] for k in self.bus.zone_id.unique()}
+        self.zone2id = {value: key for key, value in self.id2zone.items()}