Backlinks for any domain via Common Crawl

Instantly share code, notes, and snippets. Star (314) You must be signed in to star a gist Fork (78) You must be signed in to fork a gist Select an option No results found Learn more about clone URLs Clone this repository at &lt;script src=&quot;https://gist.github.com/retlehs/cf0ac6c74476e766fba2f14076fff501.js&quot;&gt;&lt;/script&gt; Save retlehs/cf0ac6c74476e766fba2f14076fff501 to your computer and use it in GitHub Desktop. DOMAIN="${1:-example.com}" RELEASE="${CC_RELEASE:-cc-main-2026-jan-feb-mar}" CACHE="${HOME}/.cache/cc-backlinks/${RELEASE}" BASE="https://data.commoncrawl.org/projects/hyperlinkgraph/${RELEASE}/domain" VERTICES="${CACHE}/domain-vertices.txt.gz" EDGES="${CACHE}/domain-edges.txt.gz" if ! command -v duckdb >/dev/null; then echo "error: duckdb not installed. Run: brew install duckdb" >&2 # Reverse domain: roots.io -> io.roots REV_DOMAIN=$(awk -F. '{for(i=NF;i>0;i--) printf "%s%s", $i, (i>1?".":"")}' <<<"$DOMAIN") local url="$1" dest="$2" if [[ -f "$dest" ]]; then return; fi echo ">> downloading $(basename "$dest") ..." >&2 curl -L --fail -C - -o "$dest" "$url" download "${BASE}/${RELEASE}-domain-vertices.txt.gz" "$VERTICES" download "${BASE}/${RELEASE}-domain-edges.txt.gz" "$EDGES" echo ">> querying backlinks to ${DOMAIN} (reversed: ${REV_DOMAIN}) ..." >&2 echo ">> first run scans ~16 GB of gzipped edges; expect several minutes" >&2 SELECT * FROM read_csv('${VERTICES}', delim='\t', header=false, columns={'id':'BIGINT','rev_domain':'VARCHAR','num_hosts':'BIGINT'}) SELECT id FROM vertices WHERE rev_domain = '${REV_DOMAIN}' SELECT from_id FROM read_csv('${EDGES}', delim='\t', header=false, columns={'from_id':'BIGINT','to_id':'BIGINT'}) WHERE to_id = (SELECT id FROM target) array_to_string(list_reverse(string_split(v.rev_domain, '.')), '.') AS linking_domain, JOIN vertices v ON v.id = i.from_id ORDER BY v.num_hosts DESC, linking_domain; Read more about Common Crawl sujay1599 commented Apr 19, 2026 #!/usr/bin/env bash set -euo pipefail # ────────────────────────────────────────────────────────────── # cc-backlinks.sh – query Common Crawl domain-level backlinks # # Usage: ./cc-backlinks.sh [domain] [top_n] # domain – domain to look up (default: example.com) # top_n – max linking domains to show (default: 100) # # Env vars: # CC_RELEASE – crawl release slug (default: cc-main-2026-jan-feb-mar) # CC_THREADS – DuckDB thread count (default: all…