From 8dedf94ce3f8f97f8c39605e6a0f54cef5ff42dd Mon Sep 17 00:00:00 2001 From: Kaan Kilinchan <kaan10@hotmail.co.uk> Date: Mon, 1 Jun 2020 16:09:26 +0100 Subject: [PATCH] Comments made --- FYP Files/Code/SAS/Counts/TravelSample.sas | 7 ++ .../Code/SAS/Counts/totalSampleAnnual.sas | 7 ++ FYP Files/Code/SAS/Date Fix/CTDdatefix.sas | 16 ++++ FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas | 28 +++++++ .../SAS/Initial Process/TravelPacInitial.sas | 23 ++++++ .../SAS/Initial Process/climateInitial.sas | 11 +++ FYP Files/Code/SAS/Joining/QuarterJoin.sas | 55 +++++++++++++- FYP Files/Code/SAS/Joining/UKinfoJoin.sas | 7 ++ FYP Files/Code/SAS/Merge/CTDFlag.sas | 14 ++++ FYP Files/Code/SAS/Merge/FlagWB.sas | 73 +++++++++++++++++-- FYP Files/Code/SAS/Misc/TravelFix.sas | 32 ++++++++ FYP Files/Code/SAS/Misc/trim.sas | 7 ++ 12 files changed, 273 insertions(+), 7 deletions(-) diff --git a/FYP Files/Code/SAS/Counts/TravelSample.sas b/FYP Files/Code/SAS/Counts/TravelSample.sas index 87a870b..4702238 100644 --- a/FYP Files/Code/SAS/Counts/TravelSample.sas +++ b/FYP Files/Code/SAS/Counts/TravelSample.sas @@ -25,6 +25,13 @@ data travel; sample = 1; run; +/*Test to see if any null values remain */ +/* +data test; + set travel(where=(sample='#NULL')); +run; +*/ + proc export data= travel dbms=csv outfile="/folders/myshortcuts/FYP_Files/completed/completed/allTravel.csv" diff --git a/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas b/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas index 595a04b..e1202a6 100644 --- a/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas +++ b/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas @@ -26,6 +26,13 @@ data travel; sample = 1; run; +/*Test to see if any null values remain */ +/* +data test; + set travel(where=(sample='#NULL')); +run; +*/ + /* Transpose the year columns to now be rows of data */ proc transpose data=travel out = travel; diff --git a/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas b/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas index 0947d5a..5368f1d 100644 --- a/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas +++ b/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas @@ -27,6 +27,22 @@ data CTDdata; full_date= translate(full_date,'30/11/', '31/11/'); run; +/*Test to see if any incorrect date values remain */ +/* +data test; + set CTDdata(where=(full_date='31/04/')); +run; +*/ +/* +data test2; + set CTDdata(where=(full_date='31/06/')); +run; +*/ +/* +data test3; + set CTDdata(where=(full_date='31/11/')); +run; +*/ /* Export the dataset into a CSV file. */ proc export data= CTDdata diff --git a/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas b/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas index 8da9f9f..d4b72d4 100644 --- a/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas +++ b/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas @@ -24,6 +24,12 @@ data CTDdata; iday= translate(iday,'1', '0'); run; +/*Test to see if any incorrect date values remain */ +/* +data test; + set CTDdata(where=(iday=0)); +run; +*/ /*data CTDdata; set CTDdata; @@ -48,6 +54,28 @@ data CTDdata; iday = '30'; run; +/*Test to see if any incorrect date values remain */ +/* +data test; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ +/* +data test2; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ +/* +data test3; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ +/* +data test4; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ + /* Convert all attributed required for the full date (day/month/year) from integers to strings. */ data CTDdata; set CTDdata; diff --git a/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas b/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas index 675b2dc..0f74684 100644 --- a/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas +++ b/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas @@ -159,6 +159,14 @@ data allTravel2; fdate = "-10-01"; run; +/*Test to see if all records have an fdate. */ +/* +data test; + set allTravel2(where=(fdate = NULL)); +run; +*/ + + /*Concatanate all date fields to create one final one with the full date. */ Data allTravel2; Set allTravel2; @@ -209,6 +217,14 @@ data allTravel2; then destination = "United States"; run; +/*Test to see if any records have an invalid country */ +/* +data test2; + set allTravel2(where=(destination = 'Irish Republic' |destination = 'China - Hong Kong' |destination = 'China - Other' + |destination = 'Cyprus EU' |destination = 'Vyprus Non EU' |destination = 'USA')); +run; +*/ + /*Import the countries dataset with long and lat values. */ PROC IMPORT DATAFILE= '/folders/myshortcuts/FYP_Files/countries2.csv' DBMS=CSV @@ -245,6 +261,13 @@ data allTravel3; sample = '1'; run; +/*Test to see if any records have a null sample */ +/* +data test2; + set allTravel3(where=(sample = '#NULL!')); +run; +*/ + /* Migrate the following datasets in to the completed datasets library. */ data comDS.allTravel; set allTravel3; diff --git a/FYP Files/Code/SAS/Initial Process/climateInitial.sas b/FYP Files/Code/SAS/Initial Process/climateInitial.sas index 7869cd5..2d19318 100644 --- a/FYP Files/Code/SAS/Initial Process/climateInitial.sas +++ b/FYP Files/Code/SAS/Initial Process/climateInitial.sas @@ -202,6 +202,17 @@ data joinedTable; flag = 'F'; run; +/*Test to see if all records have been flagged. */ +/* +data test; + set joinedTable(where=(flag = 'T')); +run; +*/ +/* +data test2; + set joinedTable(where=(flag = 'F')); +run; +*/ /*Export the final dataset to a csv file. */ proc export data= joinedTable diff --git a/FYP Files/Code/SAS/Joining/QuarterJoin.sas b/FYP Files/Code/SAS/Joining/QuarterJoin.sas index 54a0e16..b7e4143 100644 --- a/FYP Files/Code/SAS/Joining/QuarterJoin.sas +++ b/FYP Files/Code/SAS/Joining/QuarterJoin.sas @@ -37,6 +37,20 @@ data travel2; drop quarterN; run; +/*Test to see if records have a quarter number */ +/* +data test; + set travel2(keep=(quarterNum)); +run; + +PROC SORT DATA=test + BY quarterNum; + NODUPKEY ; +RUN ; + +*/ + + /* Export the datasets. */ proc export data= travel2 dbms=csv @@ -79,6 +93,19 @@ data CTD2; quarter = 4; run; +/*Test to see if records have a quarter number */ +/* +data test2; + set CTD2(keep=(quarter)); +run; + +PROC SORT DATA=test2 + BY quarter; + NODUPKEY ; +RUN ; + +*/ + /* Due to memory concerns all unneeded columns must be removed from the dataset.*/ data ctd2; set ctd2 (keep = eventid iyear imonth country country_txt region region_txt latitude longitude attacktype1 @@ -136,6 +163,19 @@ data climate2; quarter = 4; run; +/*Test to see if records have a quarter number */ +/* +data test2; + set climate2(keep=(quarter)); +run; + +PROC SORT DATA=test2 + BY quarter; + NODUPKEY ; +RUN ; + +*/ + /* Export the dataset */ proc export data= climate2 dbms=csv @@ -191,7 +231,20 @@ data exchanges2; year = year(Date); run; - + +/*Test to see if records have a quarter number */ +/* +data test2; + set exchanges2(keep=(quarter)); +run; + +PROC SORT DATA=test2 + BY quarter; + NODUPKEY ; +RUN ; + +*/ + /* Export the dataset */ proc export data= exchanges2 dbms=csv diff --git a/FYP Files/Code/SAS/Joining/UKinfoJoin.sas b/FYP Files/Code/SAS/Joining/UKinfoJoin.sas index 3e4e4d2..b600e96 100644 --- a/FYP Files/Code/SAS/Joining/UKinfoJoin.sas +++ b/FYP Files/Code/SAS/Joining/UKinfoJoin.sas @@ -182,6 +182,13 @@ data WHOjoin; temp = Temperature____Celsius_ + 30; run; +/*Test to see if any records have a climate less than zero */ +/* +data test; + set WHOjoin(where=(temp < 0)); +run; + +*/ /* Export the dataset into a CSV. */ proc export data= WHOJoin diff --git a/FYP Files/Code/SAS/Merge/CTDFlag.sas b/FYP Files/Code/SAS/Merge/CTDFlag.sas index 22e86a7..e28ebb5 100644 --- a/FYP Files/Code/SAS/Merge/CTDFlag.sas +++ b/FYP Files/Code/SAS/Merge/CTDFlag.sas @@ -74,3 +74,17 @@ proc export data= MergeUnemployment outfile="/folders/myshortcuts/FYP_Files/completed/CTDdata.csv" replace; run; + + + +/*Test to see if all records have been flagged. */ +/* +data test; + set unemployment2(where=(flag = 'T')); +run; +*/ +/* +data test2; + set unemployment2(where=(flag = 'F')); +run; +*/ diff --git a/FYP Files/Code/SAS/Merge/FlagWB.sas b/FYP Files/Code/SAS/Merge/FlagWB.sas index 7ed446e..e2f6c91 100644 --- a/FYP Files/Code/SAS/Merge/FlagWB.sas +++ b/FYP Files/Code/SAS/Merge/FlagWB.sas @@ -75,8 +75,17 @@ proc export data= MergeUnemployment run; - - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -157,7 +166,17 @@ proc export data= MergeUnemployment run; - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -238,6 +257,17 @@ proc export data= MergeUnemployment replace; run; +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -318,7 +348,17 @@ proc export data= MergeUnemployment replace; run; - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -399,8 +439,17 @@ proc export data= MergeUnemployment replace; run; - - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -479,3 +528,15 @@ proc export data= MergeUnemployment outfile="/folders/myshortcuts/FYP_Files/completed/allExchanges.csv" replace; run; + +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ diff --git a/FYP Files/Code/SAS/Misc/TravelFix.sas b/FYP Files/Code/SAS/Misc/TravelFix.sas index 8ee278a..540aed9 100644 --- a/FYP Files/Code/SAS/Misc/TravelFix.sas +++ b/FYP Files/Code/SAS/Misc/TravelFix.sas @@ -24,6 +24,14 @@ data travel; sex = 'D/K'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (sex = 'D/K')); +run; + +*/ + /* Replace the null values with the mean of all non null values. */ data travel; set travel; @@ -31,6 +39,14 @@ data travel; visits = '2669.701048'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (visits = '#NULL!')); +run; + +*/ + /* Replace the null values with the mean of all non null values. */ data travel; set travel; @@ -38,6 +54,14 @@ data travel; nights = '24052.91091'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (nights = '#NULL!')); +run; + +*/ + /* Replace the null values with the mean of all non null values. */ data travel; set travel; @@ -45,6 +69,14 @@ data travel; spend = '1270362.926'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (spend = '#NULL!')); +run; + +*/ + /*Export the data into a CSV file */ proc export data= travel dbms=csv diff --git a/FYP Files/Code/SAS/Misc/trim.sas b/FYP Files/Code/SAS/Misc/trim.sas index d93b168..2018609 100644 --- a/FYP Files/Code/SAS/Misc/trim.sas +++ b/FYP Files/Code/SAS/Misc/trim.sas @@ -89,6 +89,13 @@ data joinedData; temp = Temperature____Celsius_ + 30; run; +/*Test to see if records have a temp < 0*/ +/* +data test; + set joinedDate(where = (temp < 0)); +run; + +*/ /* Export the dataset into a CSV. */ proc export data= joinedData dbms=csv -- GitLab