diff --git a/mlab/1.0.0/Dockerfile b/mlab/1.0.0/Dockerfile new file mode 100644 index 00000000..e2f4812f --- /dev/null +++ b/mlab/1.0.0/Dockerfile @@ -0,0 +1,13 @@ +FROM frikky/shuffle:app_sdk AS base + +FROM base AS builder +RUN mkdir /install +WORKDIR /install +COPY requirements.txt /requirements.txt +RUN pip install --prefix="/install" -r /requirements.txt + +FROM base +COPY --from=builder /install /usr/local +COPY src /app +WORKDIR /app +CMD ["python", "app.py", "--log-level", "DEBUG"] diff --git a/mlab/1.0.0/README.md b/mlab/1.0.0/README.md new file mode 100644 index 00000000..5ef89c58 --- /dev/null +++ b/mlab/1.0.0/README.md @@ -0,0 +1,51 @@ +## mlab.sh App +[mlab.sh](https://mlab.sh) app for Shuffle: domain, IP, crypto, file, hash, URL, email, phone and MAC scanning, IOC extraction, CVE intelligence and threat-actor data. + +## Actions + +| No. | Action | Description | Parameters | +|-----|--------|-------------|------------| +|1 | scan_domain | Launch a domain scan. With `wait_for_completion=true` (default) it polls until done and returns the full results (subdomains, DNS, SSL, security.txt). | **domain**, wait_for_completion, timeout +|2 | get_domain_status | Status of a domain scan | **domain** +|3 | get_domain_results | Results of a finished domain scan | **domain** +|4 | get_domain_ssl | SSL certificates seen for a domain | **domain** +|5 | capture_network_requests | Load a page and capture every network request it makes | **page_url** +|6 | lookup_ip | Geolocation, ASN and ownership for an IPv4/IPv6 address | **ip** +|7 | lookup_crypto | Sanctions, labels and risk score for a crypto address (chain blank = auto-detect) | **address**, chain +|8 | bulk_lookup_crypto | Look up many addresses, sent in batches of 100 | ***addresses***, chain +|9 | lookup_hash | Known-good / known-malicious verdict for an MD5, SHA-1, SHA-256 or SHA-512 | **hash** +|10 | bulk_lookup_hash | Look up many hashes, sent in batches of 500 | ***hashes*** +|11 | upload_file | Upload a Shuffle file (max 10 MB) for analysis, returns its sha256 | **file_id** +|12 | get_file_results | Analysis results for an uploaded file | **sha256** +|13 | get_file_tool_output | Raw output of one tool listed in the file results | **sha256**, **tool** +|14 | analyze_url | Phishing shapes, embedded redirects and what mlab knows about the host | **target_url**, resolve +|15 | analyze_email | Mailbox type, spoofability of the domain and risk score | **email** +|16 | analyze_phone | Validity, line type, operator and scam shapes (E.164) | **number** +|17 | lookup_mac | Vendor, randomization, virtualization and every notation | **mac** +|18 | extract_iocs | Pull every indicator out of raw text, with optional SMS threat scoring (`fast` / `deep`) | **text**, risk, country +|19 | get_quota | Remaining daily quota for a scan type (domain / ip / file / crypto) | **scan_type** +|20 | search_cves | Search CVEs by keyword, vendor or product | **query**, severity, published_after, exact, kev_only +|21 | get_cve | Full CVE detail including EPSS and KEV | **cve_id** +|22 | get_latest_cves | Vulnerabilities from the last 7 days | +|23 | list_threat_actors | List / search threat actors | origin, motivation, sector, limit, offset +|24 | get_threat_actor | A threat actor by slug, with aliases, tools, CVEs and techniques | **slug** +|25 | get_actors_by_cve | Threat actors known to exploit a CVE | **cve_id** + +__Note__: +- apikey and url are used for authentication (url defaults to `https://mlab.sh/api/v1`, only change it for self-hosted instances). +- CVE and threat-actor actions call the public `vuln.mlab.sh` and `actors.mlab.sh` APIs and ignore the key. +- **Bold** parameters are required. +- ***Bold italic*** parameters take a list separated by commas, spaces or new lines. +- Actions return the raw mlab.sh JSON. On an HTTP error they return `{"success": false, "status": , "error": }`. + +## Requirements + +1. An mlab.sh account. +2. An API key: **mlab.sh → Account → Settings → API Keys** (starts with `mlab_`). + +## Example workflows + +- **IP enrichment:** `lookup_ip` → `get_quota` (scan_type `ip`). +- **Domain recon:** `scan_domain` (wait_for_completion `true`) → `get_domain_ssl` → `get_quota` (scan_type `domain`). +- **Threat context for a CVE:** `get_cve` → `get_actors_by_cve` with `$get_cve.id`. +- **Phishing triage:** `extract_iocs` on an email body → `analyze_url` / `bulk_lookup_hash` on what it found. diff --git a/mlab/1.0.0/api.yaml b/mlab/1.0.0/api.yaml new file mode 100644 index 00000000..23fa448d --- /dev/null +++ b/mlab/1.0.0/api.yaml @@ -0,0 +1,514 @@ +app_version: 1.0.0 +name: mlab +description: "mlab.sh: domain / IP / crypto / file / hash / URL / email / phone / MAC scanning, IOC extraction, CVE intelligence and threat-actor data." +contact_info: + name: "mlab.sh" + url: https://mlab.sh + email: support@mlab.sh +tags: + - Intel + - Scanner + - CVE +categories: + - Intel +authentication: + required: true + parameters: + - name: apikey + description: "mlab.sh API key (starts with mlab_). Account → Settings → API Keys. CVE and threat-actor actions don't use it." + example: "mlab_..." + required: true + schema: + type: string + - name: url + description: "Core API base URL. Only change for self-hosted instances." + example: "https://mlab.sh/api/v1" + required: false + schema: + type: string +actions: + - name: scan_domain + description: "Launch a domain scan; optionally wait and return full results (subdomains, DNS, SSL, security.txt)" + parameters: + - name: domain + description: "Domain to scan" + required: true + multiline: false + example: "example.com" + schema: + type: string + - name: wait_for_completion + description: "Poll until the scan finishes and return results" + required: false + multiline: false + example: "true" + schema: + type: string + options: + - "true" + - "false" + - name: timeout + description: "Max seconds to wait" + required: false + multiline: false + example: "120" + schema: + type: string + returns: + schema: + type: string + - name: get_domain_status + description: "Status of a domain scan" + parameters: + - name: domain + description: "Domain" + required: true + multiline: false + example: "example.com" + schema: + type: string + returns: + schema: + type: string + - name: get_domain_results + description: "Results of a finished domain scan" + parameters: + - name: domain + description: "Domain" + required: true + multiline: false + example: "example.com" + schema: + type: string + returns: + schema: + type: string + - name: get_domain_ssl + description: "SSL certificates seen for a domain" + parameters: + - name: domain + description: "Domain" + required: true + multiline: false + example: "example.com" + schema: + type: string + returns: + schema: + type: string + - name: capture_network_requests + description: "Load a page and capture every network request it makes" + parameters: + - name: page_url + description: "Page URL to load" + required: true + multiline: false + example: "https://example.com" + schema: + type: string + returns: + schema: + type: string + - name: lookup_ip + description: "Geolocation, ASN and ownership for an IPv4/IPv6 address" + parameters: + - name: ip + description: "IP address" + required: true + multiline: false + example: "8.8.8.8" + schema: + type: string + returns: + schema: + type: string + - name: lookup_crypto + description: "Sanctions, labels and risk score for a crypto address" + parameters: + - name: address + description: "Wallet address" + required: true + multiline: false + example: "bc1q..." + schema: + type: string + - name: chain + description: "Chain (blank = auto-detect)" + required: false + multiline: false + example: "" + schema: + type: string + options: + - "" + - "btc" + - "eth" + - "sol" + - "trx" + - "bsc" + - "polygon" + - "arbitrum" + - "optimism" + - "base" + - "avax" + - "doge" + - "ton" + returns: + schema: + type: string + - name: bulk_lookup_crypto + description: "Look up many crypto addresses (batches of 100)" + parameters: + - name: addresses + description: "Addresses separated by commas, spaces or new lines" + required: true + multiline: true + example: "" + schema: + type: string + - name: chain + description: "Chain applied to the whole batch (blank = auto-detect)" + required: false + multiline: false + example: "" + schema: + type: string + options: + - "" + - "btc" + - "eth" + - "sol" + - "trx" + - "bsc" + - "polygon" + - "arbitrum" + - "optimism" + - "base" + - "avax" + - "doge" + - "ton" + returns: + schema: + type: string + - name: lookup_hash + description: "Known-good / known-malicious verdict for an MD5, SHA-1, SHA-256 or SHA-512" + parameters: + - name: hash + description: "Hex digest" + required: true + multiline: false + example: "44d88612fea8a8f36de82e1278abb02f" + schema: + type: string + returns: + schema: + type: string + - name: bulk_lookup_hash + description: "Look up many hashes (batches of 500)" + parameters: + - name: hashes + description: "Hashes separated by commas, spaces or new lines" + required: true + multiline: true + example: "" + schema: + type: string + returns: + schema: + type: string + - name: upload_file + description: "Upload a Shuffle file (max 10 MB) for analysis; returns its sha256" + parameters: + - name: file_id + description: "Shuffle file ID" + required: true + multiline: false + example: "$exec.file_id" + schema: + type: string + returns: + schema: + type: string + - name: get_file_results + description: "Analysis results for an uploaded file" + parameters: + - name: sha256 + description: "SHA-256 returned by upload_file" + required: true + multiline: false + example: "" + schema: + type: string + returns: + schema: + type: string + - name: get_file_tool_output + description: "Raw output of one tool listed in the file results" + parameters: + - name: sha256 + description: "SHA-256 of the file" + required: true + multiline: false + example: "" + schema: + type: string + - name: tool + description: "Tool name as listed in the results" + required: true + multiline: false + example: "yara" + schema: + type: string + returns: + schema: + type: string + - name: analyze_url + description: "Phishing shapes, embedded redirects and what mlab knows about the host" + parameters: + - name: target_url + description: "URL to analyze" + required: true + multiline: false + example: "https://bit.ly/3xYz" + schema: + type: string + - name: resolve + description: "Follow redirects / unmask short links" + required: false + multiline: false + example: "false" + schema: + type: string + options: + - "false" + - "true" + returns: + schema: + type: string + - name: analyze_email + description: "Mailbox type, spoofability of the domain and risk score" + parameters: + - name: email + description: "Email address" + required: true + multiline: false + example: "name@email.com" + schema: + type: string + returns: + schema: + type: string + - name: analyze_phone + description: "Validity, line type, operator and scam shapes" + parameters: + - name: number + description: "Phone number in E.164 form" + required: true + multiline: false + example: "+33612345678" + schema: + type: string + returns: + schema: + type: string + - name: lookup_mac + description: "Vendor, randomization, virtualization and every notation" + parameters: + - name: mac + description: "MAC address" + required: true + multiline: false + example: "00:1A:2B:3C:4D:5E" + schema: + type: string + returns: + schema: + type: string + - name: extract_iocs + description: "Pull every indicator out of raw text, with optional SMS threat scoring" + parameters: + - name: text + description: "Raw text (report, log, email, SMS), up to 1 MB" + required: true + multiline: true + example: "" + schema: + type: string + - name: risk + description: "SMS threat scoring (blank = off, fast = offline, deep = network checks)" + required: false + multiline: false + example: "" + schema: + type: string + options: + - "" + - "fast" + - "deep" + - name: country + description: "Keyword and brand pack used by the scorer" + required: false + multiline: false + example: "fr" + schema: + type: string + returns: + schema: + type: string + - name: get_quota + description: "Remaining daily quota for a scan type" + parameters: + - name: scan_type + description: "Scan type" + required: true + multiline: false + example: "domain" + schema: + type: string + options: + - "domain" + - "ip" + - "file" + - "crypto" + returns: + schema: + type: string + - name: search_cves + description: "Search CVEs by keyword, vendor or product (public, no key used)" + parameters: + - name: query + description: "Search query" + required: true + multiline: false + example: "log4j" + schema: + type: string + - name: severity + description: "Severity filter" + required: false + multiline: false + example: "" + schema: + type: string + options: + - "" + - "CRITICAL" + - "HIGH" + - "MEDIUM" + - "LOW" + - name: published_after + description: "Only CVEs published on/after (YYYY-MM-DD)" + required: false + multiline: false + example: "2024-01-01" + schema: + type: string + - name: exact + description: "Exact match" + required: false + multiline: false + example: "false" + schema: + type: string + options: + - "false" + - "true" + - name: kev_only + description: "Only CISA Known Exploited Vulnerabilities" + required: false + multiline: false + example: "false" + schema: + type: string + options: + - "false" + - "true" + returns: + schema: + type: string + - name: get_cve + description: "Full CVE detail including EPSS and KEV (public)" + parameters: + - name: cve_id + description: "CVE ID" + required: true + multiline: false + example: "CVE-2021-44228" + schema: + type: string + returns: + schema: + type: string + - name: get_latest_cves + description: "Vulnerabilities from the last 7 days (public)" + returns: + schema: + type: string + - name: list_threat_actors + description: "List / search threat actors (public)" + parameters: + - name: origin + description: "Country of origin" + required: false + multiline: false + example: "Russia" + schema: + type: string + - name: motivation + description: "Motivation" + required: false + multiline: false + example: "Espionage" + schema: + type: string + - name: sector + description: "Targeted sector" + required: false + multiline: false + example: "Government" + schema: + type: string + - name: limit + description: "Max results" + required: false + multiline: false + example: "50" + schema: + type: string + - name: offset + description: "Results to skip" + required: false + multiline: false + example: "0" + schema: + type: string + returns: + schema: + type: string + - name: get_threat_actor + description: "A threat actor by slug, with aliases, tools, CVEs and techniques (public)" + parameters: + - name: slug + description: "Actor slug" + required: true + multiline: false + example: "apt28" + schema: + type: string + returns: + schema: + type: string + - name: get_actors_by_cve + description: "Threat actors known to exploit a CVE (public)" + parameters: + - name: cve_id + description: "CVE ID" + required: true + multiline: false + example: "CVE-2021-44228" + schema: + type: string + returns: + schema: + type: string +large_image: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAAOY0lEQVR4nO2dP3MbxxmHFx6XHOHYgCWAGXn4CaSLe4apnEqEClIVJo1A9k5sNRGYJi7iYc0qUMGhKnmcmSjfgPwEFFCEVUw0FHTXIxNtuLO8w52PuN13d+/9PYPxQKAknujfs+/+udttLZdLAQBXvnB9AQC4BAIA1kAAwBoIAFgDAQBrIABgDQQArIEAgDUQALAGAgDWQADAGggAWAMBAGsgAGANBACsgQCANRAAsAYCANZAAMAaCABYAwEAayAAYA0EAKyBAIA1EACwBgIA1kAAwBoIAFgDAQBrIABgDQQArIEAgDUQALAGAgDWQADAGggAWAMBHDOdzVxfAmsggDPSND08OnxxcPDmzRt3V8GdL11fAFPSNP32j9/+4+efhRAnJydCiP39fdcXxRFUAJfp73S2hBDz+c3JyQnqgBMggMu2fz6/gQNuQReIlDRNh8Ph+/fvNzcjIUSnszWf38gvyTqAvhAxqABu0n97+1EIcXV1pb4qZUBfiJjWcrmk/p4sybT9t7cf5Zs8nc7W0dERxsQ0oAKQpl99ojRQn8jBgOwLvXv3DmNiGiAAUfovLy82NyPV+cmjBsRCiOvra/SFaMAgmLrt1ynqBUkZMCYmABXAevrP356rT2QRKOr8qPdqdujVq+/RF7IKBsEU6W8/ai8+LdqP2kUjYH0+NAPGxFZBBSBq+4UQi0+L29uPsu1XbyQq/aoC6H8Q4wF7QADrbb/6XH+fGQDoU0B6NZDv5/Mb9IUsAQHstv2LT4uS36yKwHx+o79f2S969er74+O/mL5e7mAMYL3nkyFTByRyerRoUkjn9esx1sgMAgFI05/XQK0M6G9W/hFpSKez9c03v//uuz+Zu3DWoAtEnX69Asisq8SrO+Ty42D11fn85qefsE5sDFQA6rZ/ZRHIrAzon68EdcAUEMBx+oviXmVIgPFAfdAFcpD+/ASRyrqqA3pZUB9m+kWYF6oPKoCb9Mu14ZWLA0Vtf9FqMepAHVABHKRfNv8y+qoOqNuE8ndJqPTnx8edzhbqQB1QARykv2RSaGXzX3KnkPoNeIZmPVABHKd/8WmhikBR50dfJ84gq0G3230ax/WvhyGoAM7SX1QNqq8KSwG63e74+Pirx49tXFLjQQXwIv3tR201LNZnfjI3jeZB+muCCuBF219eB0rmhf4+maDtrwMeiaxEmqY7OzsXlxfCJvqAWFWAojEx0m8EdIF8Sb+cElUD4pKR8dXV1ZMnT9H2GwEVwJf0qwqglgjU+8ztoru7uxj1mgJjAF/Sn2flIvH29jbafoOgC+Rv+mUvSPWLZPqPjo4w6jUIBPAx/XkWnxYy/XgczCwQwNP0Zx4mfvrkKdJvAwjgY/r1AbGp9Kdpij228mAWyMf06xWg1+0aSb/cnxTnD2RZgjuSJInjuPVFy59XHMeTyaTm/6IkSQaDQfSZfr9X/y9sEhDA3/QbCatMf7/f6/d7cCAPBGhy+pfLpWz7VfrhQAYMgu32++Vtng/9U71ut/6DjmmaPn/+XO7Mrt9SKhfUsN/o/1nyxmrbH0XReHz8YTp90Lcw2POR16BXAPlG/XfCfjzAWgDbPZ/x+Fh+o+oO2Eh/FEX6+4wJ47uL5AlfAcjSX90Bs+lXVUj+zTLumQoQRZGRiaZwYSoAcfqrOGAp/ZkmP98LinjXAY4COEl/uQMG5/tV7nUBZNu/sgL0eY8H2AngMP1FDtho+/Od/qIxQMTbAV4CPHRCxnj685fhMP0S5mtkjATwJP36xZjt+WQEqK5B/74PrBzg8kTYdDZ7cXBg7y63138eP/TQijRN//PLLzWfbinZq0IdTal/uPKk7s37H25vb/M5g4OFALbTv/ds7+zsTPi6U4syIe+DuP8IsmJzMxoO/8DBgebfCmE7/UKI6+vr6WwmfN2nKJN+9aSlKEi/PM71hx/+yuJMvmWjsd3vV684jj9Mp2T/rpX9/vxLjgQy79XwoFWwPKz/svHrAw0XoEpK6rz0hJE5UDH9RaPhjAP9+6HPy9BsBxreBRofH/e6XZondy8uL14cHNjuCz3oJFa9968uWP+w1+1mxsTyd+pHVvZ6Fn+A7lk2nclk0u/3CHpBBHXgQW1/eUEo7wspGj8l2nwB5EigAQ7UT7+uwcr0t+7fPdH49HMRgMYBq+MBg+nPV4MoN1bm0PbzEiBoB2ykv1yJCY/08xKAuC8kb7Wv7wBZ+lt3l80n/ewECHE8QNn293n0+xlNg+b56vHjf77/l9W5UX0i8urqimButD6LzxOjHI8cXrIkrPEAQRHo82v7JUwFoHRA/ncwGHjrQMSs36/DVwDUgRb79AcsgLzLrf5tKsR9Id/qQGSi7R+Pj8MtICLoezyN3KoV1ryQQQciE+kfHY76/V64e6uI0O9wDsUB/W7kmg6MDkdepb8f8v5CogH39zOsAzUdMJv+fsgOiGY83RKKA5nxQJIk9A7YSH//7hWcA6JJz3YF4YD+onegfvqTJClKv9pkJSAHRJOebAyuDkRRROmAkfQPPh+3od81vfIVigOiec/1BuGArgGNA6bSH2lkNpbLlIIgHPBdgPWyGIQDlHXAVPpbd39bRoCiXUf9d0A0ck+HUPpCNHXARttfssViWA74K0DN/AXkgNU6YLbtb616hjif+8xXfXbAUwFMJY95HTCe/pb2NytWdooyNcFbB3wUwGzm2NYBe+lv3R/A5Jt8vVPk+byQdwLYSBtDB+oHLqmW/pLTx/KzQx464JcA9nLGbV6o5nOYSYX0q1+uLAL59PvpgEcC2E5YEA7owarpAMFj+FHpBltFpcArB3zZHn06m/1u97f/vr62+l3W2MXfyXVK2o/au7u7p6enGxsbwr9Np9t3zz0XnUKwuRnpH+qfePTw8dIDKFtWbvNClFuwRAWj4ZWDY0/qgHsBiO8/C8gBlSoaBxITGxBV7wh54oDglv4QHSCoA0nttr9kcSBvgj/rA4Jh+sNygKAvlJjefO5XJ0b1L7k9f0DwTH9YDuh7qxh3ILGQfl2DlSMBf86pF2zTH5wD8mXWgcTaxqP63Gj5grHbvpDgnH75Gh2OePaFEvJNp4uQvSMndUAwTz/nOkCQ/szJA+U4qQMONsftdLaET7Qftesfg0Wz565afjp/ez4cDtM0FX6z0M5QyxzPmvmqDMbTOKa+xAYfXVqxiTLY6hD3hWrurZIQnjyQv4MoczQT8Tmz7gfBPjhgY1NYmn+aClBADrTum6CPkl2l3/E0qFsH7G2JTPlPk+vEIToQGXI47IUwVw7Y3hCcuA4E50Dr7uU2/V7cCkHvAM12+AT/NIP7TicuHHCefvcC0DtAeRgE6kDL7/R7IQClA/RHodA7UGd9IKGqA56k3xcByDoMTtbbiUuc/w7E3qTfIwFsB8XtMVhh1YGlzUVir9LvlwD2gmIq/ZPJZO1gwYGWf+n3TgAbQTGV/tHhqGbjGlZfaGm6DniYfh8FMBsUg+k3Eiy2dSD2Mv2eCmAqKDbSXz9YDOtA7Gv6/RWgflAspT+4OlB/jWxZzwGf09/k7dGtph91oNWI9PsugJyWfqgDBOkPrg44GQ/E3qc/AAEe6gBl+oOoAwbPpVw+xIEg0h+GANUdIE6/kWA1sg7EgaQ/GAGqOOAq/QwdGJX+iAJKf0gClDvgNv2hOGDkUst/UGGlPzABihzwIf0MxwOj3I8ruPSHJ8DKM9t8SD/qQBxg+oMUQHfAt/Sz7QvFYaY/VAGkA6PDkYfpZ+jAeHwcaPoDFsAUNtIfogMfgk1wTVgLYC/9ITqQuDiPzDl8BbCd/hAd+MCvDjAVgCb99XefhgO24SgAZfrr73gMB6zCToCw0u/EgTjYOc014CVAiOmXoA5YgpEA4abfyXNkMY86wEWA0NNPv4Vei8e8EAsBKLd9tX3OD8YDZmm+AE1Kv5PzB8ZOz/G1TcMFaEbPx6EDg6avEDs4JI+Sr3/zNcHBdUKIXrf7+vV4f3+f4HvJM/mOjo5sf5e9Z3unp6cbGxuiwSybzmQysX1wHf35ngQVYDAYLBnQfAFsO4D0Bw0LAew5YCT98tmGihOOaPvNwkUAG4f4mkq/7MxUWXhC+o3DSACzDphNv3yVO0BwCveAR7+frwCmYmQj/eWH5iL9lmAnQP0wWUp/SR1A+u3BUYA6kbKd/rwDSL9VmAqwXrBo0q87QLCIMeDX79fhK8BDHaBMv/qOSL9tWAtQ3QH69BO8Brzbfgl3Aao4gPQ3GAjwKwtMSH+zgQBlDiD9jQcCFDqA9HMAAqx2AOlnAgTI8mE6HQwGmPNhQut/EgDTpGm6s7NzcXnhyY9279ne2dmZ66vwkYY/EukEpD8gIIBhkP6wgAAmQfqDAwIYYzqbod8fHBDADNPZ7MXBgT+j3pcvX2LUWwXMAjUw/ZjzqQ4qQF2Q/qCBALVA+kMHAjQn/ej3rwHGAA1JP/r964EKsCY//vg3pL8BoAKsSZqmw+Hw/O25cA3a/jqgAqzJxsbG6enp3rM94RT0+2uCChBwHUDbXx9UgFDrANJvBAgQpAPo+ZgCXaDw+kJo+w2CChBYHUD6zQIBQnIA6TcOBAjGAaTfBhAgDAeQfktAgAAcQPrtAQF8dwDptwoE8NoBpN82EMBfB5B+AiCApw4g/TRAAB8dQPrJgADeOYD0UwIB/HIA6ScGAnjkANJPDwTwxQGk3wkQwAsHkH5XQAD3DiD9DvnS5TcX3B2Q77GLrUPwRBhgDbpAgDUQALAGAgDWQADAGggAWAMBAGsgAGANBACsgQCANRAAsAYCANZAAMAaCABYAwEAayAAYA0EAKyBAIA1EACwBgIA1kAAwBoIAFgDAQBrIABgDQQArIEAgDUQALAGAgDWQADAGggAWAMBAGsgAGANBACsgQCANRAAsAYCANZAAMAaCAAEZ/4LdlfwBTnOs0AAAAAASUVORK5CYII= diff --git a/mlab/1.0.0/requirements.txt b/mlab/1.0.0/requirements.txt new file mode 100644 index 00000000..41244e13 --- /dev/null +++ b/mlab/1.0.0/requirements.txt @@ -0,0 +1,2 @@ +requests +shuffle_sdk diff --git a/mlab/1.0.0/src/app.py b/mlab/1.0.0/src/app.py new file mode 100644 index 00000000..48f81bbf --- /dev/null +++ b/mlab/1.0.0/src/app.py @@ -0,0 +1,200 @@ +import re +import time + +import requests +from shuffle_sdk import AppBase + +CVE_BASE_URL = "https://vuln.mlab.sh/api/v1" +ACTORS_BASE_URL = "https://actors.mlab.sh/api/v1" +DEFAULT_URL = "https://mlab.sh/api/v1" + + +def split_list(raw): + """Split a pasted list on commas, whitespace or new lines, dropping blanks and duplicates.""" + return list(dict.fromkeys(v for v in re.split(r"[\s,;]+", raw or "") if v)) + + +def is_true(value): + return str(value).strip().lower() in ("true", "1", "yes") + + +def respond(r): + try: + body = r.json() + except ValueError: + body = r.text + if r.ok: + return body + return {"success": False, "status": r.status_code, "error": body} + + +class Mlab(AppBase): + __version__ = "1.0.0" + app_name = "mlab" + + def __init__(self, redis, logger, console_logger=None): + super().__init__(redis, logger, console_logger) + + # --- HTTP helpers -------------------------------------------------------- + + def _core(self, apikey, url, method, path, params=None, json=None, files=None): + base = (url or DEFAULT_URL).rstrip("/") + # The upload endpoint lives at the site root, not under /api/v1. + if path.startswith("/upload/"): + base = re.sub(r"/api/v1$", "", base) + r = requests.request( + method, + base + path, + params=params, + json=json, + files=files, + headers={"Authorization": "token " + apikey}, + timeout=120, + ) + return respond(r) + + def _public(self, base, path, params=None): + return respond(requests.get(base + path, params=params, timeout=60)) + + # --- Domain --------------------------------------------------------------- + + def scan_domain(self, apikey, domain, wait_for_completion="true", timeout="120", url=DEFAULT_URL): + launch = self._core(apikey, url, "POST", "/scan/domain", json={"domain": domain}) + if not is_true(wait_for_completion) or (isinstance(launch, dict) and launch.get("success") is False): + return launch + + deadline = time.time() + int(timeout or 120) + while time.time() < deadline: + status = self._core(apikey, url, "GET", "/scan/domain/status", params={"domain": domain}) + state = status.get("status", status.get("state")) if isinstance(status, dict) else None + if state == "success": + return self._core(apikey, url, "GET", "/scan/domain/results", params={"domain": domain}) + if state == "error" or state is False: + return {"success": False, "error": 'Domain scan failed for "%s"' % domain, "status": status} + time.sleep(4) + return { + "success": False, + "error": 'Domain scan for "%s" did not finish within %ss. Use get_domain_results later.' % (domain, timeout), + } + + def get_domain_status(self, apikey, domain, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/domain/status", params={"domain": domain}) + + def get_domain_results(self, apikey, domain, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/domain/results", params={"domain": domain}) + + def get_domain_ssl(self, apikey, domain, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/domain/ssl", params={"domain": domain}) + + def capture_network_requests(self, apikey, page_url, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/domain/loadnetworkrequest", params={"url": page_url}) + + # --- IP / crypto / hash --------------------------------------------------- + + def lookup_ip(self, apikey, ip, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/ip", params={"ip": ip}) + + def lookup_crypto(self, apikey, address, chain="", url=DEFAULT_URL): + params = {"address": address} + if chain: + params["chain"] = chain + return self._core(apikey, url, "GET", "/scan/crypto", params=params) + + def bulk_lookup_crypto(self, apikey, addresses, chain="", url=DEFAULT_URL): + return self._bulk(apikey, url, "/scan/crypto", "addresses", split_list(addresses), 100, chain) + + def lookup_hash(self, apikey, hash, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/hash", params={"hash": hash}) + + def bulk_lookup_hash(self, apikey, hashes, url=DEFAULT_URL): + return self._bulk(apikey, url, "/scan/hash", "hashes", split_list(hashes), 500) + + def _bulk(self, apikey, url, path, field, values, size, chain=""): + """Send values in as few requests as the API allows; one response per batch.""" + out = [] + for start in range(0, len(values), size): + body = {field: values[start:start + size]} + if chain: + body["chain"] = chain + out.append(self._core(apikey, url, "POST", path, json=body)) + return out[0] if len(out) == 1 else out + + # --- File ----------------------------------------------------------------- + + def upload_file(self, apikey, file_id, url=DEFAULT_URL): + f = self.get_file(file_id) + files = {"file": (f.get("filename") or "file", f["data"])} + return self._core(apikey, url, "POST", "/upload/file", files=files) + + def get_file_results(self, apikey, sha256, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/file/results", params={"sha256": sha256}) + + def get_file_tool_output(self, apikey, sha256, tool, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/file/output", params={"sha256": sha256, "tool": tool}) + + # --- URL / email / phone / MAC / IOC ---------------------------------------- + + def analyze_url(self, apikey, target_url, resolve="false", url=DEFAULT_URL): + params = {"url": target_url} + if is_true(resolve): + params["resolve"] = "true" + return self._core(apikey, url, "GET", "/scan/url", params=params) + + def analyze_email(self, apikey, email, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/email", params={"email": email}) + + def analyze_phone(self, apikey, number, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/phone", params={"number": number}) + + def lookup_mac(self, apikey, mac, url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/scan/mac", params={"mac": mac}) + + def extract_iocs(self, apikey, text, risk="", country="", url=DEFAULT_URL): + params = {} + if risk: + params["risk"] = risk + if country: + params["country"] = country + return self._core(apikey, url, "POST", "/scan/ioc", params=params, json={"text": text}) + + def get_quota(self, apikey, scan_type="domain", url=DEFAULT_URL): + return self._core(apikey, url, "GET", "/limit/" + scan_type) + + # --- CVE (public) --------------------------------------------------------- + + def search_cves(self, apikey, query, severity="", published_after="", exact="false", kev_only="false", url=DEFAULT_URL): + params = {"q": query} + if severity: + params["severity"] = severity + if published_after: + params["dateStart"] = published_after[:10] + if is_true(exact): + params["exact"] = 1 + if is_true(kev_only): + params["kev"] = 1 + return self._public(CVE_BASE_URL, "/cve", params) + + def get_cve(self, apikey, cve_id, url=DEFAULT_URL): + return self._public(CVE_BASE_URL, "/cve/" + requests.utils.quote(cve_id.strip().upper())) + + def get_latest_cves(self, apikey, url=DEFAULT_URL): + return self._public(CVE_BASE_URL, "/cve/latest") + + # --- Threat actors (public) --------------------------------------------------- + + def list_threat_actors(self, apikey, origin="", motivation="", sector="", limit="50", offset="0", url=DEFAULT_URL): + params = {"limit": limit or 50, "offset": offset or 0} + for k, v in (("origin", origin), ("motivation", motivation), ("sector", sector)): + if v: + params[k] = v + return self._public(ACTORS_BASE_URL, "/actors", params) + + def get_threat_actor(self, apikey, slug, url=DEFAULT_URL): + return self._public(ACTORS_BASE_URL, "/actors/" + requests.utils.quote(slug.strip())) + + def get_actors_by_cve(self, apikey, cve_id, url=DEFAULT_URL): + return self._public(ACTORS_BASE_URL, "/cves/%s/actors" % requests.utils.quote(cve_id.strip().upper())) + + +if __name__ == "__main__": + Mlab.run()