diff --git a/FYP Files/Code/SAS/Counts/TravelSample.sas b/FYP Files/Code/SAS/Counts/TravelSample.sas index 87a870b86b0c7092818ce1d5485f470fbfb4aa7d..4702238ea9bc62857b26693d5243667ad7a8e431 100644 --- a/FYP Files/Code/SAS/Counts/TravelSample.sas +++ b/FYP Files/Code/SAS/Counts/TravelSample.sas @@ -25,6 +25,13 @@ data travel; sample = 1; run; +/*Test to see if any null values remain */ +/* +data test; + set travel(where=(sample='#NULL')); +run; +*/ + proc export data= travel dbms=csv outfile="/folders/myshortcuts/FYP_Files/completed/completed/allTravel.csv" diff --git a/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas b/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas index 595a04b652760feeeb28534ccb4d733b8b93b4d0..e1202a62523fdeb9ea6e69921792aa2862d0ceb1 100644 --- a/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas +++ b/FYP Files/Code/SAS/Counts/totalSampleAnnual.sas @@ -26,6 +26,13 @@ data travel; sample = 1; run; +/*Test to see if any null values remain */ +/* +data test; + set travel(where=(sample='#NULL')); +run; +*/ + /* Transpose the year columns to now be rows of data */ proc transpose data=travel out = travel; diff --git a/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas b/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas index 0947d5a88b78dffabc11eba29087f8b8b923529c..5368f1dcdf97852074ae3624600db429710cbf54 100644 --- a/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas +++ b/FYP Files/Code/SAS/Date Fix/CTDdatefix.sas @@ -27,6 +27,22 @@ data CTDdata; full_date= translate(full_date,'30/11/', '31/11/'); run; +/*Test to see if any incorrect date values remain */ +/* +data test; + set CTDdata(where=(full_date='31/04/')); +run; +*/ +/* +data test2; + set CTDdata(where=(full_date='31/06/')); +run; +*/ +/* +data test3; + set CTDdata(where=(full_date='31/11/')); +run; +*/ /* Export the dataset into a CSV file. */ proc export data= CTDdata diff --git a/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas b/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas index 8da9f9f03603603c20aa826c8f874bbc003ad20b..d4b72d4cbb219a72b1c8572ba1f65eeb1c7b3b7c 100644 --- a/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas +++ b/FYP Files/Code/SAS/Date Fix/CTDdatefix2.sas @@ -24,6 +24,12 @@ data CTDdata; iday= translate(iday,'1', '0'); run; +/*Test to see if any incorrect date values remain */ +/* +data test; + set CTDdata(where=(iday=0)); +run; +*/ /*data CTDdata; set CTDdata; @@ -48,6 +54,28 @@ data CTDdata; iday = '30'; run; +/*Test to see if any incorrect date values remain */ +/* +data test; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ +/* +data test2; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ +/* +data test3; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ +/* +data test4; + set CTDdata(where=(iday='31' & imonth = '04')); +run; +*/ + /* Convert all attributed required for the full date (day/month/year) from integers to strings. */ data CTDdata; set CTDdata; diff --git a/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas b/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas index 675b2dc83a3dce1b153944484a3bd600a5ea0c69..0f74684955560ea8c16c5b115ee5be35cb910c5d 100644 --- a/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas +++ b/FYP Files/Code/SAS/Initial Process/TravelPacInitial.sas @@ -159,6 +159,14 @@ data allTravel2; fdate = "-10-01"; run; +/*Test to see if all records have an fdate. */ +/* +data test; + set allTravel2(where=(fdate = NULL)); +run; +*/ + + /*Concatanate all date fields to create one final one with the full date. */ Data allTravel2; Set allTravel2; @@ -209,6 +217,14 @@ data allTravel2; then destination = "United States"; run; +/*Test to see if any records have an invalid country */ +/* +data test2; + set allTravel2(where=(destination = 'Irish Republic' |destination = 'China - Hong Kong' |destination = 'China - Other' + |destination = 'Cyprus EU' |destination = 'Vyprus Non EU' |destination = 'USA')); +run; +*/ + /*Import the countries dataset with long and lat values. */ PROC IMPORT DATAFILE= '/folders/myshortcuts/FYP_Files/countries2.csv' DBMS=CSV @@ -245,6 +261,13 @@ data allTravel3; sample = '1'; run; +/*Test to see if any records have a null sample */ +/* +data test2; + set allTravel3(where=(sample = '#NULL!')); +run; +*/ + /* Migrate the following datasets in to the completed datasets library. */ data comDS.allTravel; set allTravel3; diff --git a/FYP Files/Code/SAS/Initial Process/climateInitial.sas b/FYP Files/Code/SAS/Initial Process/climateInitial.sas index 7869cd50dd54b8e5255a364fee79b82b6d6456d3..2d19318e6895cfc16f40fc53b6e76d8035968274 100644 --- a/FYP Files/Code/SAS/Initial Process/climateInitial.sas +++ b/FYP Files/Code/SAS/Initial Process/climateInitial.sas @@ -202,6 +202,17 @@ data joinedTable; flag = 'F'; run; +/*Test to see if all records have been flagged. */ +/* +data test; + set joinedTable(where=(flag = 'T')); +run; +*/ +/* +data test2; + set joinedTable(where=(flag = 'F')); +run; +*/ /*Export the final dataset to a csv file. */ proc export data= joinedTable diff --git a/FYP Files/Code/SAS/Joining/QuarterJoin.sas b/FYP Files/Code/SAS/Joining/QuarterJoin.sas index 54a0e161c03e92d58ad4a71d834698122bc3d1d1..b7e41433a151e826e6e9037e01f16ae922c6bb41 100644 --- a/FYP Files/Code/SAS/Joining/QuarterJoin.sas +++ b/FYP Files/Code/SAS/Joining/QuarterJoin.sas @@ -37,6 +37,20 @@ data travel2; drop quarterN; run; +/*Test to see if records have a quarter number */ +/* +data test; + set travel2(keep=(quarterNum)); +run; + +PROC SORT DATA=test + BY quarterNum; + NODUPKEY ; +RUN ; + +*/ + + /* Export the datasets. */ proc export data= travel2 dbms=csv @@ -79,6 +93,19 @@ data CTD2; quarter = 4; run; +/*Test to see if records have a quarter number */ +/* +data test2; + set CTD2(keep=(quarter)); +run; + +PROC SORT DATA=test2 + BY quarter; + NODUPKEY ; +RUN ; + +*/ + /* Due to memory concerns all unneeded columns must be removed from the dataset.*/ data ctd2; set ctd2 (keep = eventid iyear imonth country country_txt region region_txt latitude longitude attacktype1 @@ -136,6 +163,19 @@ data climate2; quarter = 4; run; +/*Test to see if records have a quarter number */ +/* +data test2; + set climate2(keep=(quarter)); +run; + +PROC SORT DATA=test2 + BY quarter; + NODUPKEY ; +RUN ; + +*/ + /* Export the dataset */ proc export data= climate2 dbms=csv @@ -191,7 +231,20 @@ data exchanges2; year = year(Date); run; - + +/*Test to see if records have a quarter number */ +/* +data test2; + set exchanges2(keep=(quarter)); +run; + +PROC SORT DATA=test2 + BY quarter; + NODUPKEY ; +RUN ; + +*/ + /* Export the dataset */ proc export data= exchanges2 dbms=csv diff --git a/FYP Files/Code/SAS/Joining/UKinfoJoin.sas b/FYP Files/Code/SAS/Joining/UKinfoJoin.sas index 3e4e4d21de51c107c78e42eab5a04f55b0b33a65..b600e96c96cd7ef956ac7eb445da408e333ae4cf 100644 --- a/FYP Files/Code/SAS/Joining/UKinfoJoin.sas +++ b/FYP Files/Code/SAS/Joining/UKinfoJoin.sas @@ -182,6 +182,13 @@ data WHOjoin; temp = Temperature____Celsius_ + 30; run; +/*Test to see if any records have a climate less than zero */ +/* +data test; + set WHOjoin(where=(temp < 0)); +run; + +*/ /* Export the dataset into a CSV. */ proc export data= WHOJoin diff --git a/FYP Files/Code/SAS/Merge/CTDFlag.sas b/FYP Files/Code/SAS/Merge/CTDFlag.sas index 22e86a7737d1ffcce9c81dd24b0b273ad396c1e9..e28ebb554707f204e5a5a4cb6befbae49cc10b4a 100644 --- a/FYP Files/Code/SAS/Merge/CTDFlag.sas +++ b/FYP Files/Code/SAS/Merge/CTDFlag.sas @@ -74,3 +74,17 @@ proc export data= MergeUnemployment outfile="/folders/myshortcuts/FYP_Files/completed/CTDdata.csv" replace; run; + + + +/*Test to see if all records have been flagged. */ +/* +data test; + set unemployment2(where=(flag = 'T')); +run; +*/ +/* +data test2; + set unemployment2(where=(flag = 'F')); +run; +*/ diff --git a/FYP Files/Code/SAS/Merge/FlagWB.sas b/FYP Files/Code/SAS/Merge/FlagWB.sas index 7ed446e9c241a83edda7e0f871bccb68480905f6..e2f6c9142e4e09db3e658164eb5a95c321d029dc 100644 --- a/FYP Files/Code/SAS/Merge/FlagWB.sas +++ b/FYP Files/Code/SAS/Merge/FlagWB.sas @@ -75,8 +75,17 @@ proc export data= MergeUnemployment run; - - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -157,7 +166,17 @@ proc export data= MergeUnemployment run; - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -238,6 +257,17 @@ proc export data= MergeUnemployment replace; run; +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -318,7 +348,17 @@ proc export data= MergeUnemployment replace; run; - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -399,8 +439,17 @@ proc export data= MergeUnemployment replace; run; - - +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ @@ -479,3 +528,15 @@ proc export data= MergeUnemployment outfile="/folders/myshortcuts/FYP_Files/completed/allExchanges.csv" replace; run; + +/*Test to see if all records have been flagged. */ +/* +data test; + set MergeUnemployment(where=(flag = 'T')); +run; +*/ +/* +data test2; + set MergeUnemployment(where=(flag = 'F')); +run; +*/ diff --git a/FYP Files/Code/SAS/Misc/TravelFix.sas b/FYP Files/Code/SAS/Misc/TravelFix.sas index 8ee278a869d66b17716ad56aa83f32a7cefa10a3..540aed96446936ee2e3a9b1aa172c5fa19e86fb4 100644 --- a/FYP Files/Code/SAS/Misc/TravelFix.sas +++ b/FYP Files/Code/SAS/Misc/TravelFix.sas @@ -24,6 +24,14 @@ data travel; sex = 'D/K'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (sex = 'D/K')); +run; + +*/ + /* Replace the null values with the mean of all non null values. */ data travel; set travel; @@ -31,6 +39,14 @@ data travel; visits = '2669.701048'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (visits = '#NULL!')); +run; + +*/ + /* Replace the null values with the mean of all non null values. */ data travel; set travel; @@ -38,6 +54,14 @@ data travel; nights = '24052.91091'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (nights = '#NULL!')); +run; + +*/ + /* Replace the null values with the mean of all non null values. */ data travel; set travel; @@ -45,6 +69,14 @@ data travel; spend = '1270362.926'; run; +/*Test to see if records have D/K */ +/* +data test; + set travel(where = (spend = '#NULL!')); +run; + +*/ + /*Export the data into a CSV file */ proc export data= travel dbms=csv diff --git a/FYP Files/Code/SAS/Misc/trim.sas b/FYP Files/Code/SAS/Misc/trim.sas index d93b168fc374c9e8bcab21bc8e31f12909b9f3b3..2018609fc0473929609af2452e3b82163304ee3b 100644 --- a/FYP Files/Code/SAS/Misc/trim.sas +++ b/FYP Files/Code/SAS/Misc/trim.sas @@ -89,6 +89,13 @@ data joinedData; temp = Temperature____Celsius_ + 30; run; +/*Test to see if records have a temp < 0*/ +/* +data test; + set joinedDate(where = (temp < 0)); +run; + +*/ /* Export the dataset into a CSV. */ proc export data= joinedData dbms=csv