Merge branch 'core'

merge core to main

README.md

@@ -67,9 +67,9 @@ In this document, you'll find `usage & installation` guide. You can check our [d
   - [https://go.dev/dl/](https://go.dev/dl/)
 - Python 3.6+
   - It is needed for OpenSquat.
-- OpenSquat
+- OpenSquat **(Optional, increases Phishing & Impersonate module performance)**
   - https://github.com/atenreiro/opensquat
-- search.censys.io API ID and API KEY
+- search.censys.io API ID and API KEY **(Optional, increases Phishing & Impersonate module performance)**
   - It's used in Phishing module and controlled in .ENV file with `CENSYS_API_SECRET` key. We'll see .ENV file in the **Configuration** section.
 - `7777` and `7778` ports should be available
 - "**google-chrome**" executable file should present in $PATH

controller/blacklist.go

@@ -10,16 +10,11 @@ import (
 
 	"github.com/Chista-Framework/Chista/helpers"
 	"github.com/Chista-Framework/Chista/logger"
+	"github.com/Chista-Framework/Chista/models"
 	"github.com/chromedp/chromedp"
 	"github.com/gin-gonic/gin"
 )
 
-type blacklst struct {
-	Status string `json:"status"`
-	Name   string `json:"name"`
-	Link   string `json:"link"`
-}
-
 const (
 	BLACKLISTURL = "https://mxtoolbox.com/blacklists.aspx"
 )
@@ -32,11 +27,18 @@ func CheckBlacklist(ctx *gin.Context) {
 		ctx.JSON(http.StatusInternalServerError, gin.H{"msg": "Cannot initilaize WebSocket connection with Client. If you want to use just HTTP API set API_ONLY=true"})
 		return
 	}
-	time.Sleep(3 * time.Second)
+	time.Sleep(1 * time.Second)
 	defer helpers.CloseWSConnection()
 
-	// Query string created.
-	userInput := ctx.Query("asset")
+	// Query string checked for the domain and IP validation by ParseGivenDomain function.
+	userDomain, err := helpers.ParseGivenDomain(ctx.Query("asset"))
+	if err != nil {
+		logger.Log.Errorln("An error occurred during input parsing:", err)
+		helpers.SendMessageWS("Blacklist", strings.ToUpper(err.Error()), "error")
+		ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		helpers.SendMessageWS("Blacklist", "chista_EXIT_chista", "info")
+		return
+	}
 
 	// Checking the verbosity condition.
 	if ctx.Query("verbosity") != "" {
@@ -56,56 +58,59 @@ func CheckBlacklist(ctx *gin.Context) {
 	helpers.VERBOSITY = verbosity
 
 	// Creattes context for http request.
-	c, cancel := chromedp.NewContext(context.Background())
-	defer cancel()
+	timeoutContext, timeoutCancel := context.WithTimeout(context.Background(), 45*time.Second)
+	defer timeoutCancel()
+
+	chromedpContext, chromedpCancel := chromedp.NewContext(timeoutContext)
+	defer chromedpCancel()
 
 	// Table variable created for scraping data table.
 	var tableHTML string
-	helpers.SendMessageWS("Blacklist", "Reaching the source to check ip/domain.", "debug")
-	logger.Log.Info("Reaching the source to check ip/domain.")
+	helpers.SendMessageWS("Blacklist", "Reaching the source to check ip/domain: "+userDomain, "debug")
+	logger.Log.Info("Reaching the source to check ip/domain: " + userDomain)
 
-	// Scrapes the url.
-	err = chromedp.Run(c,
+	// Scrapes the website.
+	err = chromedp.Run(chromedpContext,
 		chromedp.Navigate(BLACKLISTURL),
 		chromedp.Click(`#ctl00_ContentPlaceHolder1_ucToolhandler_txtToolInput`, chromedp.ByID),
-		chromedp.SendKeys(`#ctl00_ContentPlaceHolder1_ucToolhandler_txtToolInput`, userInput, chromedp.ByID),
+		chromedp.SendKeys(`#ctl00_ContentPlaceHolder1_ucToolhandler_txtToolInput`, userDomain, chromedp.ByID),
 		chromedp.KeyEvent("\r"),
 		chromedp.Sleep(5*time.Second),
 		chromedp.OuterHTML(`tbody`, &tableHTML, chromedp.ByQuery),
 	)
 	if err != nil {
 		logger.Log.Errorln("An error occurred during reaching the source", err)
-		ctx.JSON(http.StatusNotFound, gin.H{"message": "An error occurred during reaching the source"})
-		helpers.SendMessageWS("Blacklist", "An error occurred during reaching the source", "error")
+		helpers.SendMessageWS("Blacklist", "An error occurred during reaching the source "+err.Error(), "error")
+		helpers.SendMessageWS("Blacklist", "chista_EXIT_chista", "info")
+		ctx.JSON(http.StatusNotFound, gin.H{"message": err.Error()})
 		return
 	}
 
 	// Filters the table data from whole response body.
-	blist := extractTableRows(tableHTML)
-	time.Sleep(3 * time.Second)
+	backlistedList := extractTableRows(tableHTML)
 
 	// Returns the filtered data.
-	if len(blist) == 0 {
+	if len(backlistedList) == 0 {
 		ctx.JSON(http.StatusOK, gin.H{"message": "IP/Domain is not blacklisted."})
 		helpers.SendMessageWS("Blacklist", fmt.Sprintln("IP/DOMAIN IS NOT BLACKLISTED."), "info")
 		helpers.SendMessageWS("Blacklist", "chista_EXIT_chista", "info")
 	} else {
-		ctx.JSON(http.StatusOK, blist)
-		for _, blacklistedSources := range blist{
-			helpers.SendMessageWS("", fmt.Sprintf("\n-------------------[%v]-------------------\nLink: %v\n%v\n", 
-			blacklistedSources.Name, blacklistedSources.Link, blacklistedSources.Status), "")
+		ctx.JSON(http.StatusOK, backlistedList)
+		for _, blacklistedSources := range backlistedList {
+			helpers.SendMessageWS("", fmt.Sprintf("\n-------------------[%v]-------------------\nLink: %v\n%v\n",
+				blacklistedSources.Name, blacklistedSources.Status, blacklistedSources.Link), "")
 		}
 		helpers.SendMessageWS("Blacklist", "chista_EXIT_chista", "info")
 	}
 
 }
 
 // Filters the table data from whole response body.
-func extractTableRows(tableHTML string) []blacklst {
-	helpers.SendMessageWS("Blacklist", "Filtering data...", "info")
+func extractTableRows(tableHTML string) []models.Blacklst {
+	helpers.SendMessageWS("Blacklist", "Filtering data...", "debug")
 	logger.Log.Debugln("Filtering the data")
 
-	lst := []blacklst{}
+	statusBlacklistedDNSList := []models.Blacklst{}
 
 	rows := strings.Split(tableHTML, "<tr>")
 	for _, row := range rows {
@@ -118,10 +123,10 @@ func extractTableRows(tableHTML string) []blacklst {
 			linkEndIndex := strings.Index(row[linkStartIndex:], "\"") + linkStartIndex
 			link := row[linkStartIndex:linkEndIndex]
 
-			l1 := blacklst{Status: "Status Blacklisted", Name: name, Link: link}
-			lst = append(lst, l1)
+			statusBlacklistedDNS := models.Blacklst{Status: "Status Blacklisted", Name: name, Link: link}
+			statusBlacklistedDNSList = append(statusBlacklistedDNSList, statusBlacklistedDNS)
 		}
 	}
 
-	return lst
+	return statusBlacklistedDNSList
 }

controller/phishing.go

@@ -21,8 +21,10 @@ import (
 
 var (
 	LevenshteinDomains_Registered []models.ResponseDomain
+	QueriedDomain                 string // It holds previous request's queried domain
 	ORIGINAL_WORKING_DIR          string
 	VERBOSITY                     int
+	isVerified                    bool
 )
 
 // TO DO
@@ -126,15 +128,20 @@ func GetPhishingDomains(ctx *gin.Context) {
 		punny_code_domains := GetPunnyCodeDomains(query_phishing_domain_model)
 		var extracted_domains_from_ct []string
 
-		// 	Check SSL CT and found websites if they don't redirect to original domain
+		// 	Check Censys API credentials set correctly
+		isVerified = helpers.IsCensysCredsSet()
+
 		for i := 0; i < len(punny_code_domains); i++ {
-			censys_hits, err := GetDomainsFromCensysCTLogs(punny_code_domains[i])
-			if err != nil {
-				logger.Log.Warnf("GetDomainsFromCensysCTLogs error :  %v", err)
-				helpers.SendMessageWS("CTLogs-Censys", fmt.Sprintf("GetDomainsFromCensysCTLogs error :  %v", err), "warn")
-			}
+			if isVerified {
+				censys_hits, err := GetDomainsFromCensysCTLogs(punny_code_domains[i])
+				if err != nil {
+					logger.Log.Warnf("GetDomainsFromCensysCTLogs error :  %v", err)
+					helpers.SendMessageWS("CTLogs-Censys", fmt.Sprintf("GetDomainsFromCensysCTLogs error :  %v", err), "warn")
+				}
 
-			extracted_domains_from_ct = append(extracted_domains_from_ct, censys_hits...)
+				extracted_domains_from_ct = append(extracted_domains_from_ct, censys_hits...)
+
+			}
 
 			crtsh_hits, err := GetDomainsFromCrtshCTLogs(punny_code_domains[i])
 			if err != nil {
@@ -648,16 +655,18 @@ func GetImpersonatingDomains(ctx *gin.Context) {
 		return
 	}
 
-	// Apply leveinsthein algortihm to generate new domains, set the treshold %33 of the provided input
-	wanted_distance := len(query_phishing_domain_model.Hostname) / 3
-	similar_domains := helpers.GenerateSimilarDomains(query_phishing_domain_model.Hostname, wanted_distance, query_phishing_domain_model.TLD)
-
 	// If LevenstheinDomains_Registered already calculated, simply return it
-	if len(LevenshteinDomains_Registered) > 0 {
+	// TO DO: The queried domain should be checked
+	if len(LevenshteinDomains_Registered) > 0 && QueriedDomain == query_phishing_domain_model.Domain {
 		ctx.JSON(http.StatusOK, &LevenshteinDomains_Registered)
 		return
 	}
 
+	// Apply leveinsthein algortihm to generate new domains, set the treshold %33 of the provided input
+	wanted_distance := len(query_phishing_domain_model.Hostname) / 3
+	similar_domains := helpers.GenerateSimilarDomains(query_phishing_domain_model.Hostname, wanted_distance, query_phishing_domain_model.TLD)
+	logger.Log.Debugf("Similar domains by levensthein: %v", similar_domains)
+
 	// [x] Check the whois records of generated domains
 	logger.Log.Infoln("Whois checker started...")
 
@@ -688,6 +697,7 @@ func GetImpersonatingDomains(ctx *gin.Context) {
 
 	// Set LevenstheinDomains_Registered for PhishingController
 	LevenshteinDomains_Registered = response_possible_ph_domains
+	QueriedDomain = query_phishing_domain_model.Domain
 
 	helpers.SendMessageWS("", "--------------------------------------------------------", "")
 	helpers.SendMessageWS("", "-------------- IMPERSONATE MODULE RESULTS --------------", "")

controller/source.go

@@ -83,8 +83,6 @@ func GetCTIData(urls string, ctx *gin.Context) {
 	// Splits parameters, arguments and fills into splitedParams.
 	for _, arg := range splitedQuery {
 		paramValue := strings.Split(arg, "=")
-		fmt.Println(paramValue[0], "=", paramValue[1])
-		fmt.Println(len(paramValue[1]))
 		if len(paramValue) == 2 {
 			splitedParams[paramValue[0]] = paramValue[1]
 		} else {