Improved GitLab data source and ported Searchcode source to API usage

README.md

@@ -24,11 +24,11 @@ The OWASP Amass Project performs network mapping of attack surfaces and external
 
 | Technique    | Data Sources |
 |:-------------|:-------------|
-| APIs         | 360PassiveDNS, Ahrefs, AnubisDB, BinaryEdge, BufferOver, BuiltWith, C99, Chaos, CIRCL, Cloudflare, DNSDB, DNSRepo, Detectify, FOFA, FullHunt, GitHub, GitLab, Greynoise, HackerTarget, Hunter, IntelX, LeakIX, Maltiverse, Mnemonic, N45HT, PassiveTotal, PentestTools, Quake, Shodan, SonarSearch, Spamhaus, Spyse, Sublist3rAPI, ThreatBook, ThreatCrowd, ThreatMiner, Twitter, URLScan, VirusTotal, ZETAlytics, ZoomEye |
+| APIs         | 360PassiveDNS, Ahrefs, AnubisDB, BinaryEdge, BufferOver, BuiltWith, C99, Chaos, CIRCL, Cloudflare, DNSDB, DNSRepo, Detectify, FOFA, FullHunt, GitHub, GitLab, Greynoise, HackerTarget, Hunter, IntelX, LeakIX, Maltiverse, Mnemonic, N45HT, PassiveTotal, PentestTools, Quake, Searchcode, Shodan, SonarSearch, Spamhaus, Spyse, Sublist3rAPI, ThreatBook, ThreatCrowd, ThreatMiner, Twitter, URLScan, VirusTotal, ZETAlytics, ZoomEye |
 | Certificates | Active pulls (optional), Censys, CertSpotter, Crtsh, Digitorus, FacebookCT, GoogleCT |
 | DNS          | Brute forcing, Reverse DNS sweeping, NSEC zone walking, Zone transfers, FQDN alterations/permutations, FQDN Similarity-based Guessing |
 | Routing      | ARIN, BGPTools, BGPView, IPdata, IPinfo, NetworksDB, RADb, Robtex, ShadowServer, TeamCymru |
-| Scraping     | AbuseIPDB, Ask, Baidu, Bing, DNSDumpster, DuckDuckGo, Gists, HackerOne, HyperStat, IPv4Info, PKey, RapidDNS, Riddler, Searchcode, Searx, SiteDossier, Yahoo |
+| Scraping     | AbuseIPDB, Ask, Baidu, Bing, DNSDumpster, DuckDuckGo, Gists, HackerOne, HyperStat, IPv4Info, PKey, RapidDNS, Riddler, Searx, SiteDossier, Yahoo |
 | Web Archives | ArchiveIt, Arquivo, CommonCrawl, HAW, UKWebArchive, Wayback |
 | WHOIS        | AlienVault, AskDNS, DNSlytics, ONYPHE, SecurityTrails, SpyOnWeb, Umbrella, WhoisXMLAPI |
 

examples/config.ini

@@ -223,15 +223,14 @@ minimum_ttl = 1440 ; One day
 #[data_sources.GitHub.accountname]
 #apikey =
 
-# https://gitlab.com (Freemium)
+# https://gitlab.com (Free)
+# GitLab apikey is the personal access token with at least read_repository or api scope
 #[data_sources.GitLab]
-#[data_sources.GitLab.free]
-#apikey =
-#[data_sources.GitLab.premium]
+#ttl = 4320
+#[data_sources.GitLab.accountname]
 #apikey =
 
 # https://hackertarget.com (Paid/Free)
-# HackerTarget can be used without an API key, but the key allows better results
 #[data_sources.HackerTarget]
 #ttl = 1440
 #[data_sources.HackerTarget.Credentials]

resources/scripts/api/gitlab.ads

@@ -1,6 +1,8 @@
 -- Copyright 2021 Jeff Foley. All rights reserved.
 -- Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 
+local json = require("json")
+
 name = "GitLab"
 type = "api"
 
@@ -32,15 +34,35 @@ function vertical(ctx, domain)
         return
     end
 
-    local scopes = {"issues", "blobs", "notes"}
-    for _, s in pairs(scopes) do
-        scrape(ctx, {
-            url=build_url(domain, s),
-            headers={['PRIVATE-TOKEN']=c.key},
+    local resp, err = request(ctx, {
+        ['url']=search_url(domain, scope),
+        ['headers']={['PRIVATE-TOKEN']=c.key},
+    })
+    if (err ~= nil and err ~= "") then
+        log(ctx, "vertical request to service failed: " .. err)
+        return
+    end
+
+    local j = json.decode(resp)
+    if (j == nil or #j == 0) then
+        return
+    end
+
+    for _, item in pairs(j) do
+        local ok = scrape(ctx, {
+            ['url']=get_file_url(item.project_id, item.path, item.ref),
+            ['headers']={['PRIVATE-TOKEN']=c.key},
         })
+        if not ok then
+            send_names(ctx, item.data)
+        end
     end
 end
 
-function build_url(domain, scope)
-    return "https://gitlab.com/api/v4/search?scope=" .. scope .. "&search=" .. domain:gsub("%.", "[.]")
+function get_file_url(id, path, ref)
+    return "https://gitlab.com/api/v4/projects/" .. id .. "/repository/files/" .. path:gsub("/", "%%2f") .. "/raw?ref=" .. ref
+end
+
+function search_url(domain)
+    return "https://gitlab.com/api/v4/search?scope=blobs&search=" .. domain
 end

resources/scripts/api/searchcode.ads

@@ -0,0 +1,22 @@
+-- Copyright 2021 Jeff Foley. All rights reserved.
+-- Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+name = "Searchcode"
+type = "api"
+
+function start()
+    set_rate_limit(2)
+end
+
+function vertical(ctx, domain)
+    for i=0,49 do
+        local ok = scrape(ctx, {['url']=build_url(domain, i)})
+        if not ok then
+            return
+        end
+    end
+end
+
+function build_url(domain, pagenum)
+    return "https://searchcode.com/api/codesearch_I/?per_page=100&q=." .. domain .. "&p=" .. pagenum
+end