From 0f651bae790f2807a9692ef41cc609bbc0685172 Mon Sep 17 00:00:00 2001 From: odinkh <odinkh@stud.ntnu.no> Date: Thu, 19 May 2022 18:42:49 +0200 Subject: [PATCH] Removed unused code and renamed functions from test->call --- api/alienvault.go | 52 +----------------- api/google.go | 111 +------------------------------------ api/hybridanalysis.go | 124 +++++------------------------------------- 3 files changed, 16 insertions(+), 271 deletions(-) diff --git a/api/alienvault.go b/api/alienvault.go index acb618d..a0e7acf 100644 --- a/api/alienvault.go +++ b/api/alienvault.go @@ -10,56 +10,6 @@ import ( "sync" ) -// CallAlienVaultUrl function takes a url, returns data on it from the alienvault api -func CallAlienVaultUrl(url string) (response utils.FrontendResponse) { - - APIKey := utils.APIKeyOTX - - getURL := "https://otx.alienvault.com//api/v1/indicators/url/" + url + "/general" - - req, err := http.NewRequest("GET", getURL, nil) - req.Header.Set("X-OTX-API-KEY", APIKey) - - //fmt.Println(req.Header) - - client := &http.Client{} - - res, err := client.Do(req) - if err != nil { - panic(err) - } - defer res.Body.Close() - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - fmt.Println("ERROR READING JSON DATA", err) - logging.Logerror(err, "ERROR READING JSON DATA, AlienVault API") - } - - var jsonResponse utils.AlienVaultURL - - err = json.Unmarshal(body, &jsonResponse) - if err != nil { - fmt.Println("UNMARSHAL ERROR:\n\n", err) - logging.Logerror(err, "Unmarshal error AlienVault API") - } - - //output:= string(body) - //fmt.Println(output) - //fmt.Println("\n\nAMOUNT OF PULSES:::::: ", jsonResponse.PulseInfo.Count) - if jsonResponse.PulseInfo.Count == 0 { - response.Status = "Safe" - } else { - response.Status = "Risk" - } - - response.SourceName = "AlienVault" - - //response = string(body) - - return response -} - // CallAlienVaultHash function takes a hash, returns data on it from the alienvault api func CallAlienVaultHash(hash string, response *utils.FrontendResponse2, wg *sync.WaitGroup) { @@ -112,7 +62,7 @@ func CallAlienVaultHash(hash string, response *utils.FrontendResponse2, wg *sync } } -func TestAlienVaultUrl(url string, response *utils.FrontendResponse2, wg *sync.WaitGroup) { +func CallAlienVaultUrl(url string, response *utils.FrontendResponse2, wg *sync.WaitGroup) { defer wg.Done() APIKey := utils.APIKeyOTX diff --git a/api/google.go b/api/google.go index 9a4d11a..d240f3e 100644 --- a/api/google.go +++ b/api/google.go @@ -13,115 +13,8 @@ import ( //"dcsg2900-threattotal/main" ) -// CallGoogleUrl function takes a url, returns data on it from the google safebrowsing api -func CallGoogleUrl(url string) (response utils.FrontendResponse) { - // Google API returnerer [] om den ikke kjenner til domenet / URL. Kan bruke dette til - // å avgjøre om det er malicious eller ikke. - - APIKey := utils.APIKeyGoogle - - postURL := "https://safebrowsing.googleapis.com/v4/threatMatches:find?key=" + APIKey - - if strings.Contains(url, "https://") { - httpsSearchURL := url - container := strings.SplitAfter(url, "https://") - - httpSearchURL := "http://" + container[1] - - fmt.Println("1 : This is the HTTP URL after splitting and concatinating", httpSearchURL) - fmt.Println("1 : This is the HTTPs URL after splitting and concatinating", httpsSearchURL) - } else if strings.Contains(url, "http://") { - httpSearchURL := url - container := strings.SplitAfter(url, "http://") - httpsSearchURL := "http://" + container[1] - - fmt.Println("2 : This is the HTTP URL after splitting and concatinating", httpSearchURL) - fmt.Println("2 : This is the HTTPs URL after splitting and concatinating", httpsSearchURL) - } else { - httpSearchURL := "http://" + url - httpsSearchURL := "https://" + url - fmt.Println("3 : This is the HTTP URL after splitting and concatinating", httpSearchURL) - fmt.Println("4 : This is the HTTPs URL after splitting and concatinating", httpsSearchURL) - } - - var jsonData = []byte(` - { - "client": { - "clientId": "threattotal", - "clientVersion": "1.5.2" - }, - "threatInfo": { - "threatTypes": ["MALWARE", "SOCIAL_ENGINEERING", "THREAT_TYPE_UNSPECIFIED", "UNWANTED_SOFTWARE","POTENTIALLY_HARMFUL_APPLICATION"], - "platformTypes": ["ANY_PLATFORM"], - "threatEntryTypes": ["URL"], - "threatEntries": [ - {"url": "https://` + url + `" }, - {"url": "http://` + url + `"} - ] - } - }`) - req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(jsonData)) - if err != nil { - fmt.Println("Error: reading sending google api request") - logging.Logerror(err, "ERROR sending REQUEST, Google API") - } - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - - client := &http.Client{} - - res, err := client.Do(req) - if err != nil { - panic(err) - } - defer res.Body.Close() - - //fmt.Println("response Status:", res.Status) - //fmt.Print("Response Headers:", res.Header) - body, err := ioutil.ReadAll(res.Body) - if err != nil { - fmt.Println("Error: reading google api response") - logging.Logerror(err, "ERROR reading google api response, Google API") - } - - var jsonResponse utils.GoogleSafeBrowsing - - err = json.Unmarshal(body, &jsonResponse) - if err != nil { - fmt.Println(err) - logging.Logerror(err, "ERROR Unmarshalling google api response, Google API") - } - output := string(body) - fmt.Println("BODY::!", output) - //fmt.Println("ThreatType::::",jsonResponse.Matches[0].ThreatType) - //fmt.Println("response Body:", string(body)) - if len(jsonResponse.Matches) != 0 { - response.Content = "This URL has been marked as malicious by Google Safebrowsing, visiting is NOT recommended" - switch jsonResponse.Matches[0].ThreatType { - case "MALWARE": - response.Status = "Risk" - - case "SOCIAL_ENGINEERING": - response.Status = "Risk" - - case "UNWANTED_SOFTWARE": - response.Status = "Risk" - - default: - response.Status = "potentially unsafe" - response.Content = "This URL has been marked as suspicious, not recommended to visit." - } - } else { - response.Status = "Safe" - response.Content = "Google safebrowsing has no data that indicates this is an unsafe URL" - } - - response.SourceName = "Google SafeBrowsing Api" - - return response -} - -func TestGoGoogleUrl(url string, response *utils.FrontendResponse2, wg *sync.WaitGroup) { +func CallGoogleUrl(url string, response *utils.FrontendResponse2, wg *sync.WaitGroup) { // Google API returnerer [] om den ikke kjenner til domenet / URL. Kan bruke dette til // å avgjøre om det er malicious eller ikke. defer wg.Done() @@ -203,6 +96,6 @@ func TestGoGoogleUrl(url string, response *utils.FrontendResponse2, wg *sync.Wai } output := string(body) fmt.Println("BODY::!", output) - + utils.SetResponeObjectGoogle(jsonResponse, response) } diff --git a/api/hybridanalysis.go b/api/hybridanalysis.go index 32e2371..598d141 100644 --- a/api/hybridanalysis.go +++ b/api/hybridanalysis.go @@ -70,107 +70,17 @@ func CallHybridAnalysisHash(hash string, response *utils.FrontendResponse2, wg * } } -// CallHybridAnalyisUrl function takes a url, returns data on it from the hybridanalysis api -// https://www.hybrid-analysis.com/docs/api/v2#/Quick%20Scan/post_quick_scan_url Documentation for contacted endpoint -func CallHybridAnalyisUrl(URL string) (VirusTotal utils.FrontendResponse, urlscanio utils.FrontendResponse) { - - APIKey := utils.APIKeyHybridAnalysis - - postURL := "https://www.hybrid-analysis.com/api/v2/quick-scan/url" - - data := url.Values{} - data.Set("scan_type", "all") //What type of scan to perform - data.Set("url", URL) //Sets URL to search - data.Set("no_share_third_party", "true") //Makes the search not accessible to 3-rd party others - data.Set("allow_community_access", "false") //Does not share search with community - - req, err := http.NewRequest("POST", postURL, strings.NewReader(data.Encode())) //Sets the new request. - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("api-key", APIKey) //Set API key - req.Header.Set("User-Agent", "Falcon Sandbox") //Set USER-AGENT, just to bypass user-agent check - See documentation on API - - client := &http.Client{} - - res, err := client.Do(req) - if err != nil { - fmt.Println("Error in request") - logging.Logerror(err, "Error in request Hybrid Analysis") - } - defer res.Body.Close() - - // res.Body.Read("finished") Her skal jeg føre en sjekk som sjekker om "finished = true eller false" - - // Hvis denne er false skal den vente 5 sekunder og kjøre requesten på nytt. - // Eventuelt om det er en måte å ikke close requesten før den er finished??????? - - // Her kan det sjekkes om VirusTotal - Status er Malicious og om Urlscan.io - // - status er malicious, suspicious, clean etc. også bare returnere denne responsen. - - //fmt.Println("response Status:", res.Status) - //fmt.Print("Response Headers:", res.Header) - body, err := ioutil.ReadAll(res.Body) - if err != nil { - fmt.Println("Ioutil error:", err) - logging.Logerror(err, "Ioutil error HybridAnalysis: ") - - } - - //var jsonData map[string]interface{} - var jsonResponse utils.HybridAnalysisURL - - err = json.Unmarshal(body, &jsonResponse) - if err != nil { - fmt.Println(err) - } - - if !jsonResponse.Finished { - time.Sleep(20 * time.Second) - - res, err := client.Do(req) - if err != nil { - panic(err) - } - defer res.Body.Close() - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - fmt.Println("Ioutil error:", err) - logging.Logerror(err, "Ioutil error HybridAnalysis: ") - - } - - var jsonResponse utils.HybridAnalysisURL - - err = json.Unmarshal(body, &jsonResponse) - if err != nil { - fmt.Println(err) - } - } - - VirusTotal.SourceName = jsonResponse.Scanners[0].Name - VirusTotal.Status = jsonResponse.Scanners[0].Status - - // Set the clean value to safe instead for frontend display. - if VirusTotal.Status == "clean" { - VirusTotal.Status = "Safe" - } - - urlscanio.SourceName = jsonResponse.Scanners[1].Name - urlscanio.Status = jsonResponse.Scanners[1].Status - - fmt.Println("Attempted HybridAnalysisURL output VT:", VirusTotal.SourceName, " Status:", VirusTotal.Status) - fmt.Println("\n\nAttempted HybridAnalysisURL output VT:", urlscanio.SourceName, " Status:", urlscanio.Status) - - return VirusTotal, urlscanio -} //Function to perform request to the Hybrid Analysis API for URL and domain intelligence. // https://www.hybrid-analysis.com/docs/api/v2#/Quick%20Scan/post_quick_scan_url Documentation on used API endpoint. -func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlscanio *utils.FrontendResponse2, wg *sync.WaitGroup) { +func CallHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlscanio *utils.FrontendResponse2, wg *sync.WaitGroup) { defer wg.Done() + VirusTotal.SourceName = "VirusTotal" //It is needed to decalre sourcenames early, incase of an unexpected error + urlscanio.SourceName = "urlscan.io" + APIKey := utils.APIKeyHybridAnalysis postURL := "https://www.hybrid-analysis.com/api/v2/quick-scan/url" @@ -188,7 +98,7 @@ func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlsc client := &http.Client{} - res, err := client.Do(req) + res, err := client.Do(req) //Do the request, if error set generic error if err != nil { fmt.Println(err, "Error in request to Hybrid Analysis - URL endpoint. ") logging.Logerror(err, "Error in request to Hybrid Analysis - URL") @@ -196,10 +106,10 @@ func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlsc utils.SetGenericError(urlscanio) return } - defer res.Body.Close() + defer res.Body.Close() //Close the responsebody. fmt.Println("response Status:", res.Status) - if res.StatusCode == http.StatusOK { + if res.StatusCode == http.StatusOK { //Chekc statuscode before continue body, err := ioutil.ReadAll(res.Body) if err != nil { @@ -227,22 +137,21 @@ func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlsc } defer res.Body.Close() - body, err := ioutil.ReadAll(res.Body) + body, err := ioutil.ReadAll(res.Body) //Read the response body if err != nil { fmt.Println("Ioutil error:", err) logging.Logerror(err, "Ioutil error HybridAnalysis: ") } - var jsonResponse utils.HybridAnalysisURL + var jsonResponse utils.HybridAnalysisURL //Declare a new struct - err = json.Unmarshal(body, &jsonResponse) + err = json.Unmarshal(body, &jsonResponse) //Unmarshal the response into strcut if err != nil { fmt.Println(err) } } - fmt.Println(jsonResponse) - VirusTotal.SourceName = jsonResponse.Scanners[0].Name + VirusTotal.SourceName = jsonResponse.Scanners[0].Name //Declaring sourcenames based on the scanner info, incase of changes. urlscanio.SourceName = jsonResponse.Scanners[1].Name utils.SetResponseObjectVirusTotal(jsonResponse, VirusTotal) @@ -253,8 +162,6 @@ func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlsc if err != nil { fmt.Println("Ioutil error:", err) logging.Logerror(err, "Ioutil error HybridAnalysis: ") - VirusTotal.SourceName = "VirusTotal" - urlscanio.SourceName = "urlscan.io" utils.SetGenericError(VirusTotal) utils.SetGenericError(urlscanio) @@ -266,21 +173,18 @@ func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlsc if err != nil { fmt.Println(err) logging.Logerror(err, "Ioutil error HybridAnalysis: ") - VirusTotal.SourceName = "VirusTotal" - urlscanio.SourceName = "urlscan.io" utils.SetGenericError(VirusTotal) utils.SetGenericError(urlscanio) } if jsonResponse.Message == "Failed to download file: domain does not exist" { //If message contains this, it means domain does not exist - VirusTotal.SourceName = "VirusTotal" + VirusTotal.EN.Status = "Safe" - VirusTotal.EN.Content = "Domain does not exist" + VirusTotal.EN.Content = "Domain does not exist" //Write the output in english and norwegian to be displayed on frontend. VirusTotal.NO.Status = "Trygg" VirusTotal.NO.Content = "Domenet eksisterer ikke" - urlscanio.SourceName = "urlscan.io" urlscanio.EN.Status = "Safe" urlscanio.EN.Content = "Domain does not exist" @@ -289,11 +193,9 @@ func TestHybridAnalyisUrl(URL string, VirusTotal *utils.FrontendResponse2, urlsc } } else { - VirusTotal.SourceName = "VirusTotal" VirusTotal.EN.Status = "Error" VirusTotal.NO.Status = "Error" - urlscanio.SourceName = "urlscan.io" urlscanio.EN.Status = "Error" urlscanio.NO.Status = "Error" } -- GitLab