update times adapter #4 #5 #27

database_adapter/reeem_adapter_times_paneu.py

@@ -5,21 +5,20 @@
 __url__ = "https://www.gnu.org/licenses/agpl-3.0.en.html"
 __author__ = "Ludwig Hülk"
 __issue__ = "https://github.com/ReeemProject/reeem_db/issues/4"
-__version__ = "v0.1.2"
+__version__ = "v0.1.3"
 
-import os
-import time
-import datetime
-import pandas as pd
 from reeem_io import *
 
 # input
 filename = "2017-11-15_Base_TIMESPanEU_FrameworkV1_DataV1_Input.xlsx"
 
-regions = ['AT']
-# regions = ['EU28', 'AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'ES', 
-#     'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 
-#     'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK', 'UK']
+#regions = ['AT']
+#regions = ['EU28', 'AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'ES',
+#    'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV',
+#    'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK', 'UK']
+regions = ['AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'ES',
+    'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV',
+    'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK', 'UK']
 
 empty_rows = 4
 
@@ -28,13 +27,9 @@
 db_table_input = 'reeem_times_paneu_input'
 db_table_output = 'reeem_times_paneu_output'
 
-def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table,
-                           region,
-                           con):
+def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, region, con):
     """read excel file and sheets, make dataframe and write to database"""
 
-    log = logger()
-
     # read file
     path = os.path.join('Model_Data', 'TIMESPanEU', filename)
     xls = pd.ExcelFile(path)
@@ -44,23 +39,23 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table,
     # make dataframe
     df.columns = ['indicator', 'unit',
                   '2015', '2020', '2025', '2030', '2035', '2040',
-                  '2045', '2050', 'field', 'category', 'aggregation', 'source']
+                  '2045', '2050', 'category', 'field', 'aggregation', 'source']
     df.index.names = ['nid']
-    # print(df.dtypes)
+    #  print(df.dtypes)
     # print(df.head())
 
     # seperate columns
-    dfunit = df[['field', 'category', 'indicator', 'unit', 'aggregation',
+    dfunit = df[['category', 'field', 'indicator', 'unit', 'aggregation',
                  'source']].copy().dropna()
     dfunit.index.names = ['nid']
-    dfunit.columns = ['field', 'category', 'indicator', 'unit', 'aggregation',
+    dfunit.columns = ['category', 'field', 'indicator', 'unit', 'aggregation',
                       'source']
     # print(dfunit)
     # print(dfunit.dtypes)
 
     # drop seperated columns
     dfclean = df.drop(
-        ['field', 'category', 'indicator', 'unit', 'aggregation', 'source'],
+        ['category', 'field', 'indicator', 'unit', 'aggregation', 'source'],
         axis=1).dropna()
     # print(dfclean)
 
@@ -75,13 +70,19 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table,
     dfdb = dfstack.join(dfunit, on='nid')
     dfdb.index.names = ['dfid']
     dfdb['pathway'] = fns['pathway']
+    dfdb['framework'] = fns['framework']
     dfdb['version'] = fns['version']
     dfdb['region'] = region
     dfdb['updated'] = fns['day']
         # (datetime.datetime.fromtimestamp(time.time())
             # .strftime('%Y-%m-%d %H:%M:%S'))
     # print(dfdb)
 
+    # i/o
+    if fns['io'] == "Input":
+        db_table = db_table_input
+    else:   db_table = db_table_output
+
     # copy dataframe to database
     dfdb.to_sql(con=con,
                 schema=db_schema,
@@ -90,6 +91,10 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table,
                 index=True)
     log.info('......sheet {} sucessfully imported...'.format(region))
 
+    # scenario log
+    scenario_log(con, 'REEEM', __version__, 'import', db_schema, db_table,
+                 os.path.basename(__file__), filename)
+
 
 if __name__ == '__main__':
     # logging
@@ -102,37 +107,18 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table,
     log.info('...model: {}'.format(fns['model']))
     log.info('...framework: {}'.format(fns['framework']))
     log.info('...version: {}'.format(fns['version']))
+    log.info('...i/o: {}'.format(fns['io']))
     log.info('...regions: {}'.format(regions))
     log.info('...establish database connection...')
 
     # connection
     con = reeem_session()
     log.info('...read file(s)...')
 
-    # # input
-    # for region in regions:
-    #     times_paneu_2_reeem_db(model, pathway, version, file_name_input,
-    #                            empty_rows,
-    #                            db_schema, db_table_input, region, con)
-    #
-    # # scenario log
-    # reeem_scenario_log(con, version, 'import', db_schema, db_table_input,
-    #                    os.path.basename(__file__), file_name_input)
-
-    # i/o
-    if fns['io'] == "import":
-        db_table = db_table_input
-    else:   db_table = db_table_output
-
-
     # import
     for region in regions:
         times_paneu_2_reeem_db(filename, fns, empty_rows,
-                               db_schema, db_table, region, con)
-
-    # scenario log
-    reeem_scenario_log(con, fns['version'], 'import', db_schema, db_table,
-                       os.path.basename(__file__), filename)
+                               db_schema, region, con)
 
     # close connection
     con.close()

database_setup/reeem_db_setup_times_paneu.sql

@@ -11,20 +11,28 @@ __license__     = "GNU Affero General Public License Version 3 (AGPL-3.0)"
 __url__         = "https://www.gnu.org/licenses/agpl-3.0.en.html"
 __author__      = "Ludwig Hülk"
 __issue__       = "https://github.com/ReeemProject/reeem_db/issues/4"
+
+ * This file is part of project REEEM (https://github.com/ReeemProject/reeem_db).
+ * It's copyrighted by the contributors recorded in the version control history:
+ * ReeemProject/reeem_db/database_setup/reeem_db_setup_times_paneu.sql
+ * 
+ * SPDX-License-Identifier: AGPL-3.0-or-later
 */
 
 
 -- TIMES PanEU input
 DROP TABLE IF EXISTS    model_draft.reeem_times_paneu_input CASCADE;
 CREATE TABLE            model_draft.reeem_times_paneu_input (
     "id"            serial NOT NULL,
-    nid             integer,
-    dfid            integer,
+    "nid"           integer,
+    "dfid"          integer,
     "pathway"       text,
+    "framework"     text,
     "version"       text,
     "schema"        text,
     "category"      text,
     "tags"          hstore,
+    "field"         text,
     "region"        text,
     "year"          smallint,
     "indicator"     text,
@@ -78,10 +86,12 @@ COMMENT ON TABLE model_draft.reeem_times_paneu_input IS
             {"name": "nid", "description": "Row id", "unit": "none"},
             {"name": "dfid", "description": "Internal dataframe id", "unit": "none"},
             {"name": "pathway", "description": "REEEM pathway", "unit": "none"},
+            {"name": "framework", "description": "REEEM framework", "unit": "none"},
             {"name": "version", "description": "REEEM version", "unit": "none"},
             {"name": "schema", "description": "1. classification", "unit": "none"},
             {"name": "category", "description": "2. classification", "unit": "none"},
             {"name": "tags", "description": "Free classification", "unit": "none"},
+            {"name": "field", "description": "Old classification", "unit": "none"},
             {"name": "region", "description": "Country or region", "unit": "none"},
             {"name": "year", "description": "Year", "unit": "none"},
             {"name": "indicator", "description": "Parameter name", "unit": "none"},
@@ -92,8 +102,8 @@ COMMENT ON TABLE model_draft.reeem_times_paneu_input IS
             {"name": "source", "description": "Data source", "unit": "none"} ] } ],
     "metadata_version": "1.3"}';
 
--- scenario log (version,io,schema_name,table_name,script_name,comment)
-SELECT reeem_scenario_log('v0.1.0','setup','model_draft','reeem_times_paneu_input','reeem_db_setup_times_paneu.sql',' ');
+-- scenario log (project,version,io,schema_name,table_name,script_name,comment)
+SELECT scenario_log('REEEM','v0.1.0','setup','model_draft','reeem_times_paneu_input','reeem_db_setup_times_paneu.sql',' ');
 
 
 -- TIMES PanEU Output

database_setup/reeem_model_data_template.sql

@@ -84,7 +84,7 @@ COMMENT ON TABLE model_draft.reeem_model_data_template IS
             {"name": "source", "description": "Data source", "unit": "none"} ] } ],
     "metadata_version": "1.3"}';
 
--- scenario log (version,io,schema_name,table_name,script_name,comment)
+-- scenario log (project,version,io,schema_name,table_name,script_name,comment)
 SELECT scenario_log('REEEM','v0.1.0','setup','model_draft','reeem_model_data_template','reeem_db_templates.sql',' ');