• R/O
  • SSH

コミット

タグ
未設定

よく使われているワード(クリックで追加)

javac++androidlinuxc#windowsobjective-ccocoa誰得qtpythonphprubygameguibathyscaphec計画中(planning stage)翻訳omegatframeworktwitterdomtestvb.netdirectxゲームエンジンbtronarduinopreviewer

コミットメタ情報

リビジョンc301b40bb10d33db90a9dfac4342c3ac5e555fb0 (tree)
日時2023-03-14 00:11:27
作者Lorenzo Isella <lorenzo.isella@gmai...>
コミッターLorenzo Isella

ログメッセージ

I am updating the file to handle the new dataset.

変更サマリ

差分

diff -r 2b80308d31bc -r c301b40bb10d R-codes/create_csv_sdmx.R
--- a/R-codes/create_csv_sdmx.R Thu Mar 09 23:41:49 2023 +0100
+++ b/R-codes/create_csv_sdmx.R Mon Mar 13 16:11:27 2023 +0100
@@ -11,6 +11,8 @@
1111 df_ini <- readRDS("../scoreboard.RDS")
1212
1313
14+stop_here <- 1
15+
1416 ## write_csv(df_ini,"scoreboard_2020_raw.csv")
1517
1618
@@ -31,21 +33,22 @@
3133 ## gdp <- readRDS("gdp.RDS")
3234
3335 ## gdp <- read_excel("scb_data.xlsx", "gdp_ameco_long_format") %>%
34-gdp <- read_excel("../LI.xlsx", "gdp_ameco_long_format") %>%
36+## gdp <- read_excel("../LI.xlsx", "gdp_ameco_long_format") %>%
3537
36- clean_data() %>%
37- rename("time_period"="expenditure_year",
38- "obs_value"="gdp_eur_bn") %>%
39- mutate(obs_value=obs_value*1e3) %>%
40- select(-x1) %>%
41- right_join(y=iso_map_eu28, by=c("member_state"="country")) %>%
42- rename("geo"="iso2") %>%
43- select(-c(member_state,iso3))
38+## clean_data() %>%
39+## rename("time_period"="expenditure_year",
40+## "obs_value"="gdp_eur_bn") %>%
41+## mutate(obs_value=obs_value*1e3) %>%
42+## select(-x1) %>%
43+## right_join(y=iso_map_eu28, by=c("member_state"="country")) %>%
44+## rename("geo"="iso2") %>%
45+## select(-c(member_state,iso3))
4446
4547 ## I add the extended MS name to the scoreboard
4648
47-df <- df_ini %>%
48- left_join(y=iso_map_eu28, by=c("member_state"="country")) %>%
49+df <- df_ini |>
50+ filter(aid_element_eur>0) |>
51+ left_join(y=iso_map_eu28, by=c("member_state"="country")) |>
4952 rename("year"="expenditure_year",
5053 "amount_spent_aid_element_in_eur_million"="aid_element_eur",
5154 "aid_instrument_name"="aid_instrument",
@@ -57,47 +60,59 @@
5760 ### Aid by objective
5861
5962
60-df_objective_up <- df %>%
61- group_by(year, iso2, scoreboard_objective) %>%
62- summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T)) %>%
63- ungroup %>%
64- left_join(y=gdp, by=c("year"="time_period",
65- "iso2"="geo")) %>%
66- mutate(pc_gdp=value/obs_value*100) %>%
67- group_by(year, iso2) %>%
63+## df_objective_up <- df |>
64+## group_by(year, iso2, scoreboard_objective) |>
65+## summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T)) |>
66+## ungroup() |>
67+## left_join(y=gdp, by=c("year"="time_period",
68+## "iso2"="geo")) |>
69+## mutate(pc_gdp=value/obs_value*100) |>
70+
71+## group_by(year, iso2) |>
72+## mutate(value=round_preserve_sum(value,2),
73+## pc_gdp=round_preserve_sum(pc_gdp,2)) |>
74+## group_modify(~ .x %>%
75+## adorn_totals("row", fill="Total")) |>
76+## ungroup() |>
77+## select(-obs_value) |>
78+## mutate(DATAFLOW="COMP:AID_SCB_OBJ(1.0)",
79+## FREQ="A") |>
80+## rename("MIO_EUR"="value",
81+## "PC_GDP"="pc_gdp") |>
82+## pivot_longer(cols=c(PC_GDP, MIO_EUR), names_to="UNIT",
83+## values_to="OBS_VALUE") |>
84+## rename("TIME_PERIOD"="year",
85+## "GEO"="iso2")
86+
87+
88+df_objective_up <- df |>
89+ group_by(year, iso2, scoreboard_objective) |>
90+ summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T),
91+ pc_gdp=sum(aid_element_as_percent_national_gdp, na.rm=T),
92+ value_adj=sum(aid_element_eur_adj, na.rm=T)) |>
93+ ungroup() |>
94+ ## left_join(y=gdp, by=c("year"="time_period",
95+ ## "iso2"="geo")) |>
96+ ## mutate(pc_gdp=value/obs_value*100) |>
97+
98+ group_by(year, iso2) |>
6899 mutate(value=round_preserve_sum(value,2),
69- pc_gdp=round_preserve_sum(pc_gdp,2)) %>%
100+ pc_gdp=round_preserve_sum(pc_gdp,2),
101+ value_adj=round_preserve_sum(value_adj,2)) |>
70102 group_modify(~ .x %>%
71- adorn_totals("row", fill="Total")) %>%
72- ungroup %>%
73- select(-obs_value) %>%
103+ adorn_totals("row", fill="Total")) |>
104+ ungroup() |>
105+ ## select(-obs_value) |>
74106 mutate(DATAFLOW="COMP:AID_SCB_OBJ(1.0)",
75- FREQ="A") %>%
107+ FREQ="A") |>
76108 rename("MIO_EUR"="value",
77- "PC_GDP"="pc_gdp") %>%
78- pivot_longer(cols=c(PC_GDP, MIO_EUR), names_to="UNIT",
79- values_to="OBS_VALUE") %>%
109+ "PC_GDP"="pc_gdp",
110+ "MIO_EUR_ADJ"="value_adj") |>
111+ pivot_longer(cols=c(PC_GDP, MIO_EUR, MIO_EUR_ADJ), names_to="UNIT",
112+ values_to="OBS_VALUE") |>
80113 rename("TIME_PERIOD"="year",
81114 "GEO"="iso2") ## %>%
82- ## mutate(scoreboard_objective=
83- ## recode(scoreboard_objective,"Research and development including innovation"= "Research and developement including innovation"))
84-
85-## Read the objective codes
86115
87-## objectives <- read_tsv("CL_OBJ_SCB+COMP+1.1.tsv") %>%
88-## clean_data() %>%
89-## mutate(name_en =
90-## recode(name_en,
91-## "Research and developement including innovation" =
92-## "Research and development including innovation"))
93-
94-
95-## objectives_revised <- objectives %>%
96-## distinct(name_en, .keep_all=T)
97-
98-
99-## new_objectives <- setdiff(df_objective_up$scoreboard_objective,
100-## objectives_revised$name_en)
101116
102117 objectives_revised <- df %>%
103118 select(scoreboard_objective) %>%
@@ -136,42 +151,77 @@
136151 df_objective_fin_save <- df_objective_fin%>%
137152 mutate(OBS_VALUE=format_col(OBS_VALUE,2,""))
138153
139-write_csv(df_objective_fin_save, "aid_scb_obj+COMP+2.1.sdmx.csv")
140-write_tsv(df_objective_fin_save, "aid_scb_obj+COMP+2.1.sdmx.tsv")
154+write_csv(df_objective_fin_save, "aid_scb_obj+COMP+3.0.sdmx.csv")
155+write_tsv(df_objective_fin_save, "aid_scb_obj+COMP+3.0.sdmx.tsv")
156+save_excel(df_objective_fin_save, "aid_scb_obj.xlsx")
157+
158+
141159
142160 #### write the revised list of objectives!
143161
144162 ## names(objectives_revised) <- c("CODE", "NAME_en")
145163
146164
147-write_csv(objectives_revised,"CL_OBJ_SCB+COMP+2.1.csv")
148-write_tsv(objectives_revised,"CL_OBJ_SCB+COMP+2.1.tsv")
165+write_csv(objectives_revised,"CL_OBJ_SCB+COMP+3.0.csv")
166+write_tsv(objectives_revised,"CL_OBJ_SCB+COMP+3.0.tsv")
167+save_excel(objectives_revised,"CL_OBJ_SCB.xlsx")
168+
169+
170+
171+### creation of a DSD file.
172+
173+#See https://ec.europa.eu/eurostat/online-help/redisstat-admin/en/TECH_B_structural/#represent-a-dsd-in-tsv
174+
175+dsd_ini <- read_csv("dsd-template.csv")
176+
177+dsd_save <- dsd_ini |>
178+ mutate(CONCEPT_SCHEME="CL_OBJ_SCB+ESTAT+3.0") |>
179+ mutate(ID=recode(ID, "INDIC_DE"="CL_OBJ_SCB"),
180+ REPRESENTATION=recode(REPRESENTATION,
181+ "INDIC_DE+ESTAT+1.0"=
182+ "CL_OBJ_SCB+3.0" ))
183+
184+
185+
186+
187+write_csv(dsd_save,"DSD_OBJ_SCB+COMP+3.0.csv")
188+write_tsv(dsd_save,"DSD_OBJ_SCB+COMP+3.0.tsv")
189+
190+
191+
192+### Done up to here
193+
194+#################################################################
149195
150196
151197 ### and I need to fix the DSD file!!!!!!!!! for aid_scb_obj+COMP+1.1.sdmx.csv
152198
153199 ### aid by instrument
154200
155-df_instrument_up <- df %>%
156- group_by(year, iso2, aid_instrument_name) %>%
157- summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T)) %>%
158- ungroup %>%
159- left_join(y=gdp, by=c("year"="time_period",
160- "iso2"="geo")) %>%
161- mutate(pc_gdp=value/obs_value*100) %>%
162- group_by(year, iso2) %>%
201+df_instrument_up <- df |>
202+ group_by(year, iso2, aid_instrument_name) |>
203+ summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T),
204+ pc_gdp=sum(aid_element_as_percent_national_gdp, na.rm=T),
205+ value_adj=sum(aid_element_eur_adj, na.rm=T)) |>
206+ ungroup() |>
207+ ## left_join(y=gdp, by=c("year"="time_period",
208+ ## "iso2"="geo")) %>%
209+ ## mutate(pc_gdp=value/obs_value*100) %>%
210+ group_by(year, iso2) |>
163211 mutate(value=round_preserve_sum(value,2),
164- pc_gdp=round_preserve_sum(pc_gdp,2)) %>%
212+ pc_gdp=round_preserve_sum(pc_gdp,2),
213+ value_adj=round_preserve_sum(value_adj,2)) |>
165214 group_modify(~ .x %>%
166- adorn_totals("row", fill="Total")) %>%
167- ungroup %>%
168- select(-obs_value) %>%
215+ adorn_totals("row", fill="Total")) |>
216+ ungroup() |>
217+ ## select(-obs_value) %>%
169218 mutate(DATAFLOW="COMP:AID_SCB_INST(1.0)",
170- FREQ="A") %>%
219+ FREQ="A") |>
171220 rename("MIO_EUR"="value",
172- "PC_GDP"="pc_gdp") %>%
173- pivot_longer(cols=c(PC_GDP, MIO_EUR), names_to="UNIT",
174- values_to="OBS_VALUE") %>%
221+ "PC_GDP"="pc_gdp",
222+ "MIO_EUR_ADJ"="value_adj") |>
223+ pivot_longer(cols=c(PC_GDP, MIO_EUR, MIO_EUR_ADJ), names_to="UNIT",
224+ values_to="OBS_VALUE") |>
175225 rename("TIME_PERIOD"="year",
176226 "GEO"="iso2")
177227
@@ -227,39 +277,60 @@
227277 write_csv(df_instrument_fin_save, "aid_scb_inst+COMP+2.1.sdmx.csv")
228278 write_tsv(df_instrument_fin_save, "aid_scb_inst+COMP+2.1.sdmx.tsv")
229279
280+dsd_save <- dsd_ini |>
281+ mutate(CONCEPT_SCHEME="CL_INST_SCB+ESTAT+3.0") |>
282+ mutate(ID=recode(ID, "INDIC_DE"="CL_INST_SCB"),
283+ REPRESENTATION=recode(REPRESENTATION,
284+ "INDIC_DE+ESTAT+1.0"=
285+ "CL_INST_SCB+3.0" ))
286+
287+
288+
289+
290+write_csv(dsd_save,"DSD_INST_SCB+COMP+3.0.csv")
291+write_tsv(dsd_save,"DSD_INST_SCB+COMP+3.0.tsv")
292+
293+
294+
230295
231296 ### aid by procedure
232297
233-df_procedure_up <- df %>%
234- group_by(year, iso2, procedure_name) %>%
235- summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T)) %>%
236- ungroup %>%
237- left_join(y=gdp, by=c("year"="time_period",
238- "iso2"="geo")) %>%
239- mutate(pc_gdp=value/obs_value*100) %>%
240- group_by(year, iso2) %>%
298+df_procedure_up <- df |>
299+ group_by(year, iso2, procedure_name) |>
300+ summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T),
301+ pc_gdp=sum(aid_element_as_percent_national_gdp, na.rm=T),
302+ value_adj=sum(aid_element_eur_adj, na.rm=T)) |>
303+ ungroup() |>
304+ ## left_join(y=gdp, by=c("year"="time_period",
305+ ## "iso2"="geo")) %>%
306+ ## mutate(pc_gdp=value/obs_value*100) %>%
307+ group_by(year, iso2) |>
241308 mutate(value=round_preserve_sum(value,2),
242- pc_gdp=round_preserve_sum(pc_gdp,2)) %>%
309+ pc_gdp=round_preserve_sum(pc_gdp,2),
310+ value_adj=round_preserve_sum(value_adj,2)) |>
243311 group_modify(~ .x %>%
244- adorn_totals("row", fill="Total")) %>%
245- ungroup %>%
246- select(-obs_value) %>%
312+ adorn_totals("row", fill="Total")) |>
313+ ungroup() |>
314+ ## select(-obs_value) |>
247315 mutate(DATAFLOW="COMP:AID_SCB_TYPE(1.0)",
248- FREQ="A") %>%
316+ FREQ="A") |>
249317 rename("MIO_EUR"="value",
250- "PC_GDP"="pc_gdp") %>%
251- pivot_longer(cols=c(PC_GDP, MIO_EUR), names_to="UNIT",
252- values_to="OBS_VALUE") %>%
318+ "PC_GDP"="pc_gdp",
319+ "MIO_EUR_ADJ"="value_adj") |>
320+ pivot_longer(cols=c(PC_GDP, MIO_EUR, MIO_EUR_ADJ), names_to="UNIT",
321+ values_to="OBS_VALUE") |>
253322 rename("TIME_PERIOD"="year",
254323 "GEO"="iso2")
255324
256325 ## procedures <- read_csv("CL_TYPE+COMP+1.0.csv") %>%
257326 ## clean_data()
258327
259-procedures <- df_procedure_up %>%
260- select(procedure_name) %>%
261- distinct %>%
262- add_row(procedure_name="Other") %>%
328+
329+
330+procedures <- df_procedure_up %>%
331+ select(procedure_name) %>%
332+ distinct() %>%
333+ add_row(procedure_name="Other") %>%
263334 move_row(which(.$procedure_name=="Total"), nrow(.)) %>%
264335 move_row(which(.$procedure_name=="Other"), nrow(.))
265336
@@ -302,8 +373,8 @@
302373 mutate(OBS_VALUE=format_col(OBS_VALUE,2,""))
303374
304375
305-write_csv(df_procedure_fin_save, "aid_scb_type+COMP+2.1.sdmx.csv")
306-write_tsv(df_procedure_fin_save, "aid_scb_type+COMP+2.1.sdmx.tsv")
376+write_csv(df_procedure_fin_save, "aid_scb_type+COMP+3.0.sdmx.csv")
377+write_tsv(df_procedure_fin_save, "aid_scb_type+COMP+3.0.sdmx.tsv")
307378
308379
309380
@@ -320,23 +391,28 @@
320391 "Transport costs of goods in eligible areas (Art. 15(2))"= "Transport costs of goods in eligible areas (Art. 15(2)(a))"
321392 )) %>%
322393 group_by(year, iso2, all_objective_names_gber_only) %>%
323- summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T)) %>%
394+ summarise(value=sum(amount_spent_aid_element_in_eur_million, na.rm=T),
395+ pc_gdp=sum(aid_element_as_percent_national_gdp, na.rm=T),
396+ value_adj=sum(aid_element_eur_adj, na.rm=T)
397+ ) %>%
324398 ungroup %>%
325- left_join(y=gdp, by=c("year"="time_period",
326- "iso2"="geo")) %>%
327- mutate(pc_gdp=value/obs_value*100) %>%
399+ ## left_join(y=gdp, by=c("year"="time_period",
400+ ## "iso2"="geo")) %>%
401+ ## mutate(pc_gdp=value/obs_value*100) %>%
328402 group_by(year, iso2) %>%
329403 mutate(value=round_preserve_sum(value,2),
330- pc_gdp=round_preserve_sum(pc_gdp,2)) %>%
404+ pc_gdp=round_preserve_sum(pc_gdp,2),
405+ value_adj=round_preserve_sum(value_adj,2)) %>%
331406 group_modify(~ .x %>%
332407 adorn_totals("row", fill="Total")) %>%
333408 ungroup %>%
334- select(-obs_value) %>%
409+ ## select(-obs_value) %>%
335410 mutate(DATAFLOW="COMP:AID_SCB_TYPE(1.0)",
336411 FREQ="A") %>%
337412 rename("MIO_EUR"="value",
338- "PC_GDP"="pc_gdp") %>%
339- pivot_longer(cols=c(PC_GDP, MIO_EUR), names_to="UNIT",
413+ "PC_GDP"="pc_gdp",
414+ "MIO_EUR_ADJ"="value_adj") %>%
415+ pivot_longer(cols=c(PC_GDP, MIO_EUR, MIO_EUR_ADJ), names_to="UNIT",
340416 values_to="OBS_VALUE") %>%
341417 rename("TIME_PERIOD"="year",
342418 "GEO"="iso2")
@@ -366,8 +442,8 @@
366442 objectives_gber <- df_gber_objective_up %>%
367443 select(all_objective_names_gber_only) %>%
368444 distinct %>%
369- move_row(which(.$all_objective_names_gber_only=="Total"), nrow(.)) %>%
370- move_row(which(.$all_objective_names_gber_only=="Other"), nrow(.))
445+ move_row(which(.$all_objective_names_gber_only=="Total"), nrow(.)) ## %>%
446+ ## move_row(which(.$all_objective_names_gber_only=="Other"), nrow(.))
371447
372448
373449
@@ -459,6 +535,7 @@
459535
460536
461537
538+if (stop_here != 1){
462539
463540
464541 ### aid ---> agri data
@@ -1466,5 +1543,5 @@
14661543 write_tsv(dsd_case_type_save,"DSD_OBJ_SCB_OBJ+COMP+2.1.tsv")
14671544
14681545
1469-
1546+}
14701547 print("So far so good")