Basic filtering of urls

api/url-intelligence.go

 package api
 
 import (
+	logging "dcsg2900-threattotal/logs"
 	"dcsg2900-threattotal/utils"
-	"dcsg2900-threattotal/logs"
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 
 	"github.com/gin-gonic/gin"
@@ -90,12 +91,18 @@ func urlSearch(url string) (data []byte, err error, complete bool) {
 	urlscanio = &responseData[2]
 	alienvault = &responseData[3]
 
+	wg.Add(2)
 	fmt.Println(url)
-
-	wg.Add(3)
+	if checkUrlAgainstFilter(url) {
+		wg.Add(1)
 		go TestGoGoogleUrl(url, p, &wg)
 		go TestHybridAnalyisUrl(url, VirusTotal, urlscanio, &wg)
 		go TestAlienVaultUrl(url, alienvault, &wg)
+	} else {
+		go giveTrueGoogleUrl(url, p)
+		go TestHybridAnalyisUrl(url, VirusTotal, urlscanio, &wg)
+		go TestAlienVaultUrl(url, alienvault, &wg)
+	}
 	wg.Wait()
 
 	var resultResponse utils.ResultFrontendResponse
@@ -132,3 +139,20 @@ func checkIfIntelligenceComplete(jsonData utils.ResultFrontendResponse, size int
 
 	return complete
 }
+
+func checkUrlAgainstFilter(url string) bool {
+	for i := 0; i < len(utils.UrlBlockList); i++ {
+		if strings.Contains(url, utils.UrlBlockList[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+func giveTrueGoogleUrl(url string, response *utils.FrontendResponse2) {
+	response.EN.Status = "Safe"
+	response.EN.Content = "Google safebrowsing has no data that indicates this is an unsafe URL/Domain"
+	response.NO.Status = "Trygg"
+	response.NO.Content = "Google Safebrowsing har ingen data som indikerer at dette er en utrygg URL/Domene"
+	response.SourceName = "Google SafeBrowsing Api"
+}

main.go

@@ -57,6 +57,11 @@ func init() {
 	utils.APIKeyGoogle = os.Getenv("APIKeyGoogle")
 	utils.APIKeyHybridAnalysis = os.Getenv("APIKeyHybridAnalysis")
 	utils.APIKeyOTX = os.Getenv("APIKeyOTX")
+
+	utils.UrlBlockList = make([]string, 3)
+	utils.UrlBlockList[0] = "ntnu.no"
+	utils.UrlBlockList[1] = "ntnu.edu"
+	utils.UrlBlockList[2] = "testsafebrowsing.appspot.com/s/malware.html"
 }
 
 func main() {

storage/redis.go

 package storage
 
 import (
+	logging "dcsg2900-threattotal/logs"
 	"dcsg2900-threattotal/utils"
-	"dcsg2900-threattotal/logs"
 	"fmt"
 	"os"
 
@@ -44,3 +44,84 @@ func AddToPool(key string, timeout int, data string) {
 	// Print the response to adding the data (should be "OK"
 	fmt.Println(response)
 }
+
+// Filter creation, creates filters for id, hash and urls
+/*
+func CreateFilters() {
+	type Test5 struct {
+		Values []string `json:"values"`
+	}
+	var test Test5
+	var hashFilter [2]string
+	var idFilter [2]string
+
+	idFilter[0] = "id1"
+	idFilter[1] = "id2"
+	hashFilter[0] = "hash1"
+	hashFilter[1] = "hash2"
+	test.Values = make([]string, 3)
+	test.Values[0] = "ntnu.edu"
+	test.Values[1] = "ntnu.no"
+	test.Values[2] = "testsafebrowsing.com/s/malware.html"
+
+	testdata, _ := json.Marshal(test)
+	fmt.Println(test)
+	_, err := utils.Conn.Do("SET", "urlFilter", testdata)
+	if err != nil {
+		fmt.Println("Error adding data to redis:" + err.Error())
+		logging.Logerror(err, "ERROR adding data to Redis, url filter list creation")
+	}
+
+	_, err = utils.Conn.Do("SETEX", "hashFilter", utils.CacheDurationFile, hashFilter)
+	if err != nil {
+		fmt.Println("Error adding data to redis:" + err.Error())
+		logging.Logerror(err, "ERROR adding data to Redis, url filter list creation")
+	}
+
+	_, err = utils.Conn.Do("SETEX", "idFilter", utils.CacheDurationFile, idFilter)
+	if err != nil {
+		fmt.Println("Error adding data to redis:" + err.Error())
+		logging.Logerror(err, "ERROR adding data to Redis, url filter list creation")
+	}
+
+}
+*/
+
+// Check filter takes a filter and an item and checks if the item is whitelisted
+/*
+func CheckFilter(filter string, item string) bool {
+
+	type test5 struct {
+		Values []string `json:"values"`
+	}
+
+	value, err := utils.Conn.Do("GET", filter)
+	if value == nil {
+		fmt.Println("No length")
+		return false
+	}
+	if err != nil {
+		return false
+	}
+
+	valueB, _ := json.Marshal(value)
+	fmt.Println(value)
+	var responseData test5
+	json.Unmarshal(valueB, &responseData)
+
+	fmt.Println(responseData)
+	test32, err := json.Marshal(responseData)
+	fmt.Printf("This is it, %q", value)
+	fmt.Println("this is test32", string(test32))
+	fmt.Println("this is test32", test32)
+	fmt.Println("Length: ", len(responseData.Values))
+	/*
+		test2, err := json.Marshal(value)
+		var test []string
+		json.Unmarshal(test2, &test)
+		fmt.Println(test2)
+
+		fmt.Sprintf("%q", value)
+*/
+//	return true
+//}

utils/global.go

@@ -21,3 +21,5 @@ var APIKeyOTX string
 var APIKeyHybridAnalysis string
 var APIKeyVirusTotal string
 var APIKeyGoogle string
+
+var UrlBlockList []string