diff --git a/database_adapter/reeem_adapter_times_paneu.py b/database_adapter/reeem_adapter_times_paneu.py index 7efb400..6a79d0e 100644 --- a/database_adapter/reeem_adapter_times_paneu.py +++ b/database_adapter/reeem_adapter_times_paneu.py @@ -5,21 +5,20 @@ __url__ = "https://www.gnu.org/licenses/agpl-3.0.en.html" __author__ = "Ludwig Hülk" __issue__ = "https://github.com/ReeemProject/reeem_db/issues/4" -__version__ = "v0.1.2" +__version__ = "v0.1.3" -import os -import time -import datetime -import pandas as pd from reeem_io import * # input filename = "2017-11-15_Base_TIMESPanEU_FrameworkV1_DataV1_Input.xlsx" -regions = ['AT'] -# regions = ['EU28', 'AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'ES', -# 'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', -# 'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK', 'UK'] +#regions = ['AT'] +#regions = ['EU28', 'AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'ES', +# 'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', +# 'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK', 'UK'] +regions = ['AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'ES', + 'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', + 'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK', 'UK'] empty_rows = 4 @@ -28,13 +27,9 @@ db_table_input = 'reeem_times_paneu_input' db_table_output = 'reeem_times_paneu_output' -def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, - region, - con): +def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, region, con): """read excel file and sheets, make dataframe and write to database""" - log = logger() - # read file path = os.path.join('Model_Data', 'TIMESPanEU', filename) xls = pd.ExcelFile(path) @@ -44,23 +39,23 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, # make dataframe df.columns = ['indicator', 'unit', '2015', '2020', '2025', '2030', '2035', '2040', - '2045', '2050', 'field', 'category', 'aggregation', 'source'] + '2045', '2050', 'category', 'field', 'aggregation', 'source'] df.index.names = ['nid'] - # print(df.dtypes) + # print(df.dtypes) # print(df.head()) # seperate columns - dfunit = df[['field', 'category', 'indicator', 'unit', 'aggregation', + dfunit = df[['category', 'field', 'indicator', 'unit', 'aggregation', 'source']].copy().dropna() dfunit.index.names = ['nid'] - dfunit.columns = ['field', 'category', 'indicator', 'unit', 'aggregation', + dfunit.columns = ['category', 'field', 'indicator', 'unit', 'aggregation', 'source'] # print(dfunit) # print(dfunit.dtypes) # drop seperated columns dfclean = df.drop( - ['field', 'category', 'indicator', 'unit', 'aggregation', 'source'], + ['category', 'field', 'indicator', 'unit', 'aggregation', 'source'], axis=1).dropna() # print(dfclean) @@ -75,6 +70,7 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, dfdb = dfstack.join(dfunit, on='nid') dfdb.index.names = ['dfid'] dfdb['pathway'] = fns['pathway'] + dfdb['framework'] = fns['framework'] dfdb['version'] = fns['version'] dfdb['region'] = region dfdb['updated'] = fns['day'] @@ -82,6 +78,11 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, # .strftime('%Y-%m-%d %H:%M:%S')) # print(dfdb) + # i/o + if fns['io'] == "Input": + db_table = db_table_input + else: db_table = db_table_output + # copy dataframe to database dfdb.to_sql(con=con, schema=db_schema, @@ -90,6 +91,10 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, index=True) log.info('......sheet {} sucessfully imported...'.format(region)) + # scenario log + scenario_log(con, 'REEEM', __version__, 'import', db_schema, db_table, + os.path.basename(__file__), filename) + if __name__ == '__main__': # logging @@ -102,6 +107,7 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, log.info('...model: {}'.format(fns['model'])) log.info('...framework: {}'.format(fns['framework'])) log.info('...version: {}'.format(fns['version'])) + log.info('...i/o: {}'.format(fns['io'])) log.info('...regions: {}'.format(regions)) log.info('...establish database connection...') @@ -109,30 +115,10 @@ def times_paneu_2_reeem_db(filename, fns, empty_rows, db_schema, db_table, con = reeem_session() log.info('...read file(s)...') - # # input - # for region in regions: - # times_paneu_2_reeem_db(model, pathway, version, file_name_input, - # empty_rows, - # db_schema, db_table_input, region, con) - # - # # scenario log - # reeem_scenario_log(con, version, 'import', db_schema, db_table_input, - # os.path.basename(__file__), file_name_input) - - # i/o - if fns['io'] == "import": - db_table = db_table_input - else: db_table = db_table_output - - # import for region in regions: times_paneu_2_reeem_db(filename, fns, empty_rows, - db_schema, db_table, region, con) - - # scenario log - reeem_scenario_log(con, fns['version'], 'import', db_schema, db_table, - os.path.basename(__file__), filename) + db_schema, region, con) # close connection con.close() diff --git a/database_setup/reeem_db_setup_times_paneu.sql b/database_setup/reeem_db_setup_times_paneu.sql index 94ce5c9..75b228c 100644 --- a/database_setup/reeem_db_setup_times_paneu.sql +++ b/database_setup/reeem_db_setup_times_paneu.sql @@ -11,6 +11,12 @@ __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" __url__ = "https://www.gnu.org/licenses/agpl-3.0.en.html" __author__ = "Ludwig Hülk" __issue__ = "https://github.com/ReeemProject/reeem_db/issues/4" + + * This file is part of project REEEM (https://github.com/ReeemProject/reeem_db). + * It's copyrighted by the contributors recorded in the version control history: + * ReeemProject/reeem_db/database_setup/reeem_db_setup_times_paneu.sql + * + * SPDX-License-Identifier: AGPL-3.0-or-later */ @@ -18,13 +24,15 @@ __issue__ = "https://github.com/ReeemProject/reeem_db/issues/4" DROP TABLE IF EXISTS model_draft.reeem_times_paneu_input CASCADE; CREATE TABLE model_draft.reeem_times_paneu_input ( "id" serial NOT NULL, - nid integer, - dfid integer, + "nid" integer, + "dfid" integer, "pathway" text, + "framework" text, "version" text, "schema" text, "category" text, "tags" hstore, + "field" text, "region" text, "year" smallint, "indicator" text, @@ -78,10 +86,12 @@ COMMENT ON TABLE model_draft.reeem_times_paneu_input IS {"name": "nid", "description": "Row id", "unit": "none"}, {"name": "dfid", "description": "Internal dataframe id", "unit": "none"}, {"name": "pathway", "description": "REEEM pathway", "unit": "none"}, + {"name": "framework", "description": "REEEM framework", "unit": "none"}, {"name": "version", "description": "REEEM version", "unit": "none"}, {"name": "schema", "description": "1. classification", "unit": "none"}, {"name": "category", "description": "2. classification", "unit": "none"}, {"name": "tags", "description": "Free classification", "unit": "none"}, + {"name": "field", "description": "Old classification", "unit": "none"}, {"name": "region", "description": "Country or region", "unit": "none"}, {"name": "year", "description": "Year", "unit": "none"}, {"name": "indicator", "description": "Parameter name", "unit": "none"}, @@ -92,8 +102,8 @@ COMMENT ON TABLE model_draft.reeem_times_paneu_input IS {"name": "source", "description": "Data source", "unit": "none"} ] } ], "metadata_version": "1.3"}'; --- scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT reeem_scenario_log('v0.1.0','setup','model_draft','reeem_times_paneu_input','reeem_db_setup_times_paneu.sql',' '); +-- scenario log (project,version,io,schema_name,table_name,script_name,comment) +SELECT scenario_log('REEEM','v0.1.0','setup','model_draft','reeem_times_paneu_input','reeem_db_setup_times_paneu.sql',' '); -- TIMES PanEU Output diff --git a/database_setup/reeem_model_data_template.sql b/database_setup/reeem_model_data_template.sql index 7fe4b2d..3bc9590 100644 --- a/database_setup/reeem_model_data_template.sql +++ b/database_setup/reeem_model_data_template.sql @@ -84,7 +84,7 @@ COMMENT ON TABLE model_draft.reeem_model_data_template IS {"name": "source", "description": "Data source", "unit": "none"} ] } ], "metadata_version": "1.3"}'; --- scenario log (version,io,schema_name,table_name,script_name,comment) +-- scenario log (project,version,io,schema_name,table_name,script_name,comment) SELECT scenario_log('REEEM','v0.1.0','setup','model_draft','reeem_model_data_template','reeem_db_templates.sql',' ');