From 17dab8d063b44ea62822402686b421e24fa3f974 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 04:48:24 +0000 Subject: [PATCH 1/4] Download the Lite and Asn files from the Distributor (get-lite-file-from-azure.ps1) --- get-lite-file-from-azure.ps1 | 96 +++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 17 deletions(-) diff --git a/get-lite-file-from-azure.ps1 b/get-lite-file-from-azure.ps1 index 08b6e9c..11e1121 100644 --- a/get-lite-file-from-azure.ps1 +++ b/get-lite-file-from-azure.ps1 @@ -7,13 +7,15 @@ param( [switch]$Asn ) -# Base URL for all data files -$BaseUrl = "https://51ddatafiles.blob.core.windows.net/enterpriseipi/" +# The Distributor's free downloads, which need no License Key. The source +# value lets downloads be counted by where they come from. +$BaseUrl = "https://distributor.51degrees.com/api/v2/free/" +$Source = "ip-intelligence-data" -# Known files: remote basename + local .ipi name +# Known files: download name + local .ipi name $KnownFiles = @( - [pscustomobject]@{ Key = "Lite"; Remote = "51Degrees-IPIV4LiteIpiV41.ipi.gz"; Name = "51Degrees-LiteV41.ipi" } - [pscustomobject]@{ Key = "Asn"; Remote = "51Degrees-IPIV4AsnIpiV41.ipi.gz"; Name = "51Degrees-IPIV4AsnIpiV41.ipi" } + [pscustomobject]@{ Key = "Lite"; Remote = "ipi-lite-v41"; Name = "51Degrees-LiteV41.ipi" } + [pscustomobject]@{ Key = "Asn"; Remote = "ipi-asn-v41"; Name = "51Degrees-IPIV4AsnIpiV41.ipi" } ) # Selection: if neither -Lite nor -Asn is passed, do all known files. @@ -25,27 +27,87 @@ $Selected = @($KnownFiles | Where-Object { }) if (-not $Selected) { $Selected = $KnownFiles } +# Unpacks the gzip data at Start into Out, and returns the length unpacked +# as a gzip member records it, modulo 2^32. +function Copy-Gzip([string]$Path, [long]$Start, $Out) { + $before = $Out.Position + $in = [File]::OpenRead($Path) + try { + $in.Position = $Start + $gzip = [Compression.GZipStream]::new($in, [Compression.CompressionMode]::Decompress) + try { $gzip.CopyTo($Out) } finally { $gzip.Dispose() } + } finally { $in.Dispose() } + [uint32](($Out.Position - $before) % 4294967296) +} + +# The length the last gzip member records, in the last 4 bytes of the file. +function Get-LastMemberSize([string]$Path) { + $in = [File]::OpenRead($Path) + try { + $tail = New-Object byte[] 4 + $in.Position = $in.Length - 4 + [void]$in.Read($tail, 0, 4) + [BitConverter]::ToUInt32($tail, 0) + } finally { $in.Dispose() } +} + +# Where the gzip member after one that unpacked to Size bytes starts, found +# as that length followed by the 1f 8b 08 that begins the next member, or +# -1 when there is none. +function Find-NextMember([string]$Path, [long]$From, [uint32]$Size) { + [byte[]]$pattern = [BitConverter]::GetBytes($Size) + [byte[]](0x1f, 0x8b, 0x08) + $in = [File]::OpenRead($Path) + try { + $buffer = New-Object byte[] (1MB) + $position = $From + while ($position -lt $in.Length) { + $in.Position = $position + $count = $in.Read($buffer, 0, $buffer.Length) + if ($count -lt $pattern.Length) { break } + $i = [Array]::IndexOf($buffer, $pattern[0], 0, $count) + while ($i -ge 0 -and $i -le $count - $pattern.Length) { + $match = $true + for ($j = 1; $j -lt $pattern.Length; $j++) { + if ($buffer[$i + $j] -ne $pattern[$j]) { $match = $false; break } + } + if ($match) { return $position + $i + 4 } + $i = [Array]::IndexOf($buffer, $pattern[0], $i + 1, $count - $i - 1) + } + $position += $count - ($pattern.Length - 1) + } + } finally { $in.Dispose() } + -1 +} + +# The Distributor sends each file as more than one gzip member. PowerShell 7 +# unpacks them all in one pass, but Windows PowerShell stops at the end of +# the first, so there each member is unpacked in turn. function Expand-GzipFile { param( [Parameter(Mandatory)] [string]$ArchiveName, [Parameter(Mandatory)] [string]$ArchivedName ) Write-Host "Extracting '$ArchiveName' to '$ArchivedName'..." + $out = [File]::Create($ArchivedName) try { - $src = [File]::OpenRead($ArchiveName) - $dest = [File]::Create($ArchivedName) - $gunzip = [Compression.GZipStream]::new($src, [Compression.CompressionMode]::Decompress) - $gunzip.copyTo($dest) - } finally { - # Avoid calling Close on nulls - if ($gunzip) { $gunzip.Close() } - if ($dest) { $dest.Close() } - if ($src) { $src.Close() } - } + if ($PSVersionTable.PSEdition -eq 'Core') { + $null = Copy-Gzip $ArchiveName 0 $out + return + } + $lastSize = Get-LastMemberSize $ArchiveName + $start = 0L + while ($true) { + $size = Copy-Gzip $ArchiveName $start $out + if ($size -eq $lastSize) { return } + # A member is at least 20 bytes, its length in the last 4. + $start = Find-NextMember $ArchiveName ($start + 16) $size + if ($start -lt 0) { throw "'$ArchiveName' did not unpack." } + } + } finally { $out.Dispose() } } foreach ($file in $Selected) { - $ArchiveUrl = $BaseUrl + $file.Remote + $ArchiveUrl = "$BaseUrl$($file.Remote)?source=$Source" $ArchivedName = (Join-Path (Get-Location) $file.Name) $ArchiveName = "$ArchivedName.gz" @@ -63,4 +125,4 @@ foreach ($file in $Selected) { # Unpack Write-Output "Extracting $ArchiveName" Expand-GzipFile -ArchiveName $ArchiveName -ArchivedName $ArchivedName -} \ No newline at end of file +} From 4c94ec05b2f08b1742c3dccd4aa94d8e00da0f53 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 04:48:28 +0000 Subject: [PATCH 2/4] Download the Lite and Asn files from the Distributor (get-lite-file-from-azure.sh) --- get-lite-file-from-azure.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/get-lite-file-from-azure.sh b/get-lite-file-from-azure.sh index 03d306f..2d0968f 100644 --- a/get-lite-file-from-azure.sh +++ b/get-lite-file-from-azure.sh @@ -1,11 +1,13 @@ #!/bin/bash -# Base URL for all data files -BASE_URL="https://51ddatafiles.blob.core.windows.net/enterpriseipi/" +# The Distributor's free downloads, which need no License Key. The source +# value lets downloads be counted by where they come from. +BASE_URL="https://distributor.51degrees.com/api/v2/free/" +SOURCE="ip-intelligence-data" -# Known files: "remote basename|local .ipi name" -LITE_FILE="51Degrees-IPIV4LiteIpiV41.ipi.gz|51Degrees-LiteV41.ipi" -ASN_FILE="51Degrees-IPIV4AsnIpiV41.ipi.gz|51Degrees-IPIV4AsnIpiV41.ipi" +# Known files: "download name|local .ipi name" +LITE_FILE="ipi-lite-v41|51Degrees-LiteV41.ipi" +ASN_FILE="ipi-asn-v41|51Degrees-IPIV4AsnIpiV41.ipi" # Default values FORCE=false @@ -48,12 +50,13 @@ fi process_file() { local remote="$1" local archived_name="$2" - local archive_url="${BASE_URL}${remote}" + local archive_url="${BASE_URL}${remote}?source=${SOURCE}" local archive_name="${archived_name}.gz" # Download if forced or archive doesn't exist if [ "$FORCE" = true ] || [ ! -f "$archive_name" ]; then - curl -o "$archive_name" "$archive_url" + # -f fails on an error answer instead of saving it as the archive. + curl -fL -o "$archive_name" "$archive_url" else echo "Archive found. Download skipped." fi From a2693dd8fa6d0320b300ec5904f4527531ff58be Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 04:48:31 +0000 Subject: [PATCH 3/4] Download the Lite and Asn files from the Distributor (tests/check-dl-script.tests.ps1) --- tests/check-dl-script.tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check-dl-script.tests.ps1 b/tests/check-dl-script.tests.ps1 index cadd570..953a9f6 100644 --- a/tests/check-dl-script.tests.ps1 +++ b/tests/check-dl-script.tests.ps1 @@ -2,7 +2,7 @@ # Pester test suite for get-lite-file-from-azure.ps1/.sh download scripts. # # Data-driven tests: one case per (script x invocation mode). Each case -# performs a REAL download from Azure blob storage into an isolated temp dir. +# performs a REAL download from the Distributor into an isolated temp dir. [CmdletBinding()] param() From 81fea6d667f36fbe79bee64e2380349956172a08 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 04:48:35 +0000 Subject: [PATCH 4/4] Download the Lite and Asn files from the Distributor get-lite-file-from-azure.ps1 and get-lite-file-from-azure.sh downloaded both files from Azure blob storage. They now use the Distributor's free downloads, ipi-lite-v41 and ipi-asn-v41, which need no License Key, and send source=ip-intelligence-data so downloads can be counted. Their options and output files are unchanged. The Distributor sends each file as more than one gzip member. The PowerShell script now unpacks every member under Windows PowerShell, whose GZipStream stops after the first, and PowerShell 7 and gunzip already read them all. The shell script passes -f to curl, so an error answer is no longer saved as the archive. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 83e4087..238fcea 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This repo allows to obtain data needed to run IP Intelligence examples: ## Download Lite data file The Lite data file is too large to commit to this repository, so one of the -scripts below downloads it from 51Degrees storage instead: +scripts below downloads it from the 51Degrees Distributor instead: | Script | Language | OS | | -- | -- | -- | @@ -66,7 +66,7 @@ This 'lite' file is built from a subsection of all IPs and is updated less frequ ### `get-lite-file-from-azure.ps1`/`.sh` -Convenience scripts to download (and unpack) data file, stored on Azure Blob Storage. +Convenience scripts to download (and unpack) the data files from the 51Degrees Distributor, which needs no License Key. ### `evidence-gen.ps1` @@ -102,7 +102,7 @@ cd ip-intelligence-data ./ci/run-unit-tests.ps1 ``` -Each test performs a **real download** from 51Degrees storage into an isolated +Each test performs a **real download** from the Distributor into an isolated temporary directory and verifies the archive is fetched, its printed MD5 matches a re-computed hash, and the file unpacks correctly. The test cases are: