Merge branch 'rest-api'

.gitignore

+json-utility

Dockerfile

+FROM golang:1.16.2-buster
+WORKDIR .
+COPY . .
+RUN make build
+RUN make build file=example.json
\ No newline at end of file

LICENSE

+MIT License
+
+Copyright (c) 2021 Ivanov, Ivan (UG - Computer Science)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

+build:
+	go get
+run:
+	go run . $(mode) $(file)
+test:
+	go test
\ No newline at end of file

README.md

-# json-utility
+# JSON Utility
+## Usage
+```
+$ make build
+$ make run mode=TEST file=example.json #TESTING WITH LOCAL FILE. Where example.json is the file we want to process
+$ make run mode=CONTAINER #RUN AS REST API. 
+```
 
+## Testing
+```
+$ make test
+```
+## Discussion
+- Designing the solution is made based on the assumption that the load will be a
+JSON file containing an array of all the entries needed to perform the calculations.
+
+## Dependencies
+- You'd need to have GOPATH exported.  GOBIN for MACOS for go get command
\ No newline at end of file

example.json

@@ -2,7 +2,7 @@
     {
         "userid": "user1",
         "url": "http://www.someamazingwebsite.com/1",
-        "type": "GET",
+        "type": "POST",
         "timestamp": 1360662163000
     },
     {
@@ -13,7 +13,7 @@
     },
     {
         "userid": "user2",
-        "url": "http://www.someamazingwebsite.com/3",
+        "url": "http://www.someamazingwebsite.com/1",
         "type": "GET",
         "timestamp": 1360662163000
     },
@@ -25,8 +25,8 @@
     },
     {
         "userid": "user2",
-        "url": "http://www.someamazingwebsite.com/2",
-        "type": "GET",
+        "url": "http://www.someamazingwebsite.com/3",
+        "type": "POST",
         "timestamp": 1360462163000
     }
 ]
\ No newline at end of file

handlers.go

+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"sync"
+)
+
+//Handles opens the JSON file and calls extractJSON to parse
+//the data passed as file. Returns the result as a struct.
+func Handles(argument string) (urls *Links, err error) {
+	jsonFile, err := os.Open(argument)
+	if err != nil {
+		log.Println(err)
+		return nil, err
+	}
+	log.Println("Successfully Opened", argument)
+	//Close when done.
+	defer jsonFile.Close()
+	payload := ExctractJSON(jsonFile)
+	result := ProcessJson(payload)
+	return result, err
+}
+
+//Web handler for requests containing JSON input.
+func Index(w http.ResponseWriter, r *http.Request) {
+	decoder := json.NewDecoder(r.Body)
+	var loads []Load
+
+	err := decoder.Decode(&loads)
+	if err != nil {
+		panic(err)
+	}
+
+	result := ProcessJson(loads)
+	WriteResult(w, result)
+}
+
+//Preocessing the JSON as a whole array.
+func ProcessJson(loads []Load) (links *Links) {
+	visited := Links{URL: make(map[string]*Dates)}
+
+	wg := new(sync.WaitGroup)
+
+	for _, load := range loads {
+		if load.TypeRequest != "GET" {
+			log.Println("Request is invalid: ", load.TypeRequest)
+			continue
+		}
+		wg.Add(1)
+		go ProcessLoad(&visited, load, wg)
+
+	}
+	wg.Wait()
+	return &visited
+}

handlerstest.go

+package main
+
+import (
+	"fmt"
+	"log"
+	"testing"
+)
+
+/*
+Standard assert Equals function
+*/
+func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
+	if a == b {
+		return
+	}
+	if len(message) == 0 {
+		message = fmt.Sprintf("%v != %v", a, b)
+	}
+	t.Fatal(message)
+}
+
+/*
+Testing with sample JSON tha websites are visited.
+*/
+func TestWebsites(t *testing.T) {
+	result, err := Handles("test/test-websites.json")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	assertEqual(t, len(result.URL), 1, "")
+}
+
+/*
+Testing if a user visits one URL multiple times but in
+different days if it's still going to add it.
+*/
+func TestWebsites2(t *testing.T) {
+	result, err := Handles("test/test-websites2.json")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	assertEqual(t, len(result.URL), 1, "")
+}
+
+/*
+Testing if each date gets added. Basic case. 1 user visits 1 page on
+2 different dates. 2 Dates should be returned. Also checks if POST
+requests get dropped.
+*/
+func TestDates(t *testing.T) {
+	result, err := Handles("test/test-dates.json")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	assertEqual(t, len(result.URL["http://www.someamazingwebsite.com/1"].Date), 2, "")
+}
+
+/*
+Testing if each date gets added. Advaced case. 3 user visits 2 page on
+3 different dates. 2 Dates should be returned for each website.
+*/
+func TestDates2(t *testing.T) {
+	result, err := Handles("test/test-dates2.json")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	assertEqual(t, len(result.URL["http://www.someamazingwebsite.com/1"].Date), 2, "")
+	assertEqual(t, len(result.URL["http://www.someamazingwebsite.com/2"].Date), 2, "")
+}

main.go

-/*
-Example payload:
-userid: A unique ID representing the user
-url : The URL the visitor has accessed
-type: The HTTP method used to access the URL
-timestamp: The timestamp for when the action occurred
-*/
+// Package implments JSON parser utility to process a stream of JSON messages
+// and calculate the number of unique viewers per day for each URL in the stream.
 package main
 
 import (
-	"encoding/json"
 	"log"
 	"net/http"
-	"time"
+	"os"
+	"sync"
 
 	"github.com/gorilla/mux"
 )
 
 type Load struct {
 	Userid      string `json:"userid"`
-	Url         string `json:"url"`
+	URL         string `json:"url"`
 	TypeRequest string `json:"type"`
 	Timestamp   int64  `json:"timestamp"`
 }
 
-type Dates struct {
-	Exists map[string]*URLs
+type Links struct {
+	URL map[string]*Dates //map holding all the Dates a certain URL is visited
+	mux sync.RWMutex      //mutex for race condition
 }
-type URLs struct {
-	Exists map[string]*URL
+type Dates struct {
+	Date map[string]*Visits //map holding all visits for certain date
 }
 
-type URL struct {
-	Counter int
-	UserIds map[string]bool
+type Visits struct {
+	mux     sync.RWMutex    //mutex for race condition
+	Counter int             //Counter for unique visits
+	UserIds map[string]bool //map holding different user ids
 }
 
 func main() {
-
-	router := mux.NewRouter().StrictSlash(true)
-	router.HandleFunc("/", Index)
-
-	log.Fatal(http.ListenAndServe(":8080", router))
-}
-
-func Index(w http.ResponseWriter, r *http.Request) {
-	decoder := json.NewDecoder(r.Body)
-
-	var loads []Load
-	dates := Dates{Exists: make(map[string]*URLs)}
-
-	err := decoder.Decode(&loads)
-	if err != nil {
-		panic(err)
-	}
-
-	for _, load := range loads {
-		if load.TypeRequest != "GET" {
-			log.Println("Request is invalid")
-			continue
+	if os.Args[1] == "CONTAINER" {
+		router := mux.NewRouter().StrictSlash(true)
+		router.HandleFunc("/", Index)
+		log.Fatal(http.ListenAndServe(":8080", router))
+	} else {
+		//Check if the usage of the script has been correct.
+		if len(os.Args) != 3 {
+			log.Println("USAGE: ./json-utility [mode] [path/to/file]")
+			return
 		}
 
-		var t string
-
-		if time.Now().UnixNano() > load.Timestamp {
-			t = time.Unix(load.Timestamp/1000, 0).Format("2006-01-02")
-		} else {
-			t = time.Unix(load.Timestamp/1000, 0).Format("2006-01-02")
+		//Get the result from a file.
+		result, err := Handles(os.Args[2])
+		if err != nil {
+			log.Println(err)
+			return
 		}
-		log.Println(t)
-		if site, ok := dates.Exists[t]; ok {
-			log.Println("Date exists.")
-
-			if url, ok := site.Exists[load.Url]; ok {
-				log.Println("URL exists.")
-				if url.UserIds[load.Userid] {
-					continue //Continue as the user has already visited that website for this day once.
-				} else {
-
-					url.Counter += 1
-					url.UserIds[load.Userid] = true
-					log.Println(url.Counter)
-				}
-			} else {
-				//Add the URL and a visit.
-				entry := URL{Counter: 0, UserIds: make(map[string]bool)}
 
-				entry.Counter += 1
-				entry.UserIds[load.Userid] = true
-				dates.Exists[t].Exists[load.Url] = &entry
-				log.Println("URL is added.")
-			}
-		} else {
-			log.Println("Date doesnt yet exist.")
-			//Initialising Date and Urls for that date.
-			entry := URL{Counter: 0, UserIds: make(map[string]bool)}
-			urls := URLs{Exists: make(map[string]*URL)}
-
-			entry.Counter += 1
-			entry.UserIds[load.Userid] = true
-			urls.Exists[load.Url] = &entry
-			dates.Exists[t] = &urls
-			log.Println("We added it though")
-		}
-		log.Println(dates)
-		//fmt.Fprintf(w, "%q", json.NewEncoder(w).Encode(load))
+		//Print the result
+		PrintResult(result)
 	}
 
-	// loop over elements of slice
-	for k, v := range dates.Exists {
-		// m is a map[string]interface.
-		// loop over keys and values in the map.
-		log.Println(k, "value is", v)
-
-	}
 }

process.go

+package main
+
+import (
+	"log"
+	"sync"
+	"time"
+)
+
+//Process handles processing a load given by
+func ProcessLoad(visited *Links, load Load, wg *sync.WaitGroup) {
+	//Notify main that this routine is done.
+	defer wg.Done()
+	//Time should be accepted both in epoch milliseconds or seconds.
+	var t string
+	if time.Now().UnixNano() > load.Timestamp {
+		t = time.Unix(load.Timestamp/1000, 0).Format("2006-01-02")
+	} else {
+		t = time.Unix(load.Timestamp, 0).Format("2006-01-02")
+	}
+
+	visited.mux.Lock()
+
+	if date, ok := visited.URL[load.URL]; ok {
+		log.Println("URL exists.")
+		if visit, ok := date.Date[t]; ok {
+			log.Println("Date exists.")
+			if visit.UserIds[load.Userid] {
+				visited.mux.Unlock()
+				return //Return as the user has already visited that website for this day once.
+			} else {
+
+				visit.mux.Lock()
+				visit.Counter += 1
+				visit.UserIds[load.Userid] = true
+
+				visited.mux.Unlock()
+				visit.mux.Unlock()
+				return
+			}
+		} else {
+			//Add the Date and a Visit.
+			entry := Visits{Counter: 0, UserIds: make(map[string]bool)}
+			entry.mux.Lock()
+			entry.Counter += 1
+			entry.UserIds[load.Userid] = true
+			entry.mux.Unlock()
+
+			visited.URL[load.URL].Date[t] = &entry
+			visited.mux.Unlock()
+
+			log.Println("Date and Visit are added.")
+			return
+		}
+	} else {
+		log.Println("URL doesn't exist.")
+		//Initialising URL, date and visits for that date.
+
+		entry := Visits{Counter: 0, UserIds: make(map[string]bool)}
+		entry.mux.Lock()
+
+		dates := Dates{Date: make(map[string]*Visits)}
+
+		entry.Counter += 1
+		entry.UserIds[load.Userid] = true
+		entry.mux.Unlock()
+
+		dates.Date[t] = &entry
+
+		visited.URL[load.URL] = &dates
+		visited.mux.Unlock()
+		return
+	}
+}

test/test-dates.json

+[
+    {
+        "userid": "user1",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1360662163000
+    },
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1160165163000
+    }
+]
\ No newline at end of file

test/test-dates2.json

+[
+    {
+        "userid": "user1",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1360662163000
+    },
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/3",
+        "type": "GET",
+        "timestamp": 1160165163000
+    },
+    {
+        "userid": "user3",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1358122163000
+    },
+    {
+        "userid": "user1",
+        "url": "http://www.someamazingwebsite.com/2",
+        "type": "GET",
+        "timestamp": 1360662163000
+    },
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/2",
+        "type": "GET",
+        "timestamp": 1160165163000
+    },
+    {
+        "userid": "user3",
+        "url": "http://www.someamazingwebsite.com/2",
+        "type": "GET",
+        "timestamp": 1358122163000
+    }
+]
\ No newline at end of file

test/test-websites.json

+[
+    {
+        "userid": "user1",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "POST",
+        "timestamp": 1360662163000
+    },
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1360662163000
+    }
+]
\ No newline at end of file

test/test-websites2.json

+[
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1360662163000
+    },
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1360662163000
+    },
+    {
+        "userid": "user2",
+        "url": "http://www.someamazingwebsite.com/1",
+        "type": "GET",
+        "timestamp": 1310666163000
+    }
+
+]
\ No newline at end of file

utils.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+)
+
+//Prints the results for a given structure containing already processed Links.
+func PrintResult(result *Links) {
+	for k, v := range result.URL {
+		log.Println("For URL: ", k)
+		for m, x := range v.Date {
+			log.Println("↳", m, "there are:", x.Counter, "unique visits.")
+		}
+	}
+}
+
+//Function for returning response to the user if the code runs in container
+//as an API
+func WriteResult(w http.ResponseWriter, result *Links) {
+	for k, v := range result.URL {
+		fmt.Fprintf(w, "For URL, %v\n", k)
+		log.Println("For URL: ", k)
+		for m, x := range v.Date {
+			fmt.Fprintf(w, "↳%v there are: %v unique visits.\n", m, x.Counter)
+			log.Println("↳", m, "there are:", x.Counter, "unique visits.")
+		}
+	}
+}
+
+//Extracts JSON payload and calls Process to handle the data.
+func ExctractJSON(file *os.File) (load []Load) {
+	d := json.NewDecoder(file)
+
+	var loads []Load
+
+	err := d.Decode(&loads)
+	if err != nil {
+		log.Panic(err)
+	}
+	return loads
+}