Download the Lite and Asn files from the Distributor

README.md

@@ -11,7 +11,7 @@ This repo allows to obtain data needed to run IP Intelligence examples:
 ## Download Lite data file
 
 The Lite data file is too large to commit to this repository, so one of the
-scripts below downloads it from 51Degrees storage instead:
+scripts below downloads it from the 51Degrees Distributor instead:
 
 | Script | Language | OS |
 | -- | -- | -- |
@@ -66,7 +66,7 @@ This 'lite' file is built from a subsection of all IPs and is updated less frequ
 
 ### `get-lite-file-from-azure.ps1`/`.sh`
 
-Convenience scripts to download (and unpack) data file, stored on Azure Blob Storage.
+Convenience scripts to download (and unpack) the data files from the 51Degrees Distributor, which needs no License Key.
 
 ### `evidence-gen.ps1`
 
@@ -102,7 +102,7 @@ cd ip-intelligence-data
 ./ci/run-unit-tests.ps1
 ```
 
-Each test performs a **real download** from 51Degrees storage into an isolated
+Each test performs a **real download** from the Distributor into an isolated
 temporary directory and verifies the archive is fetched, its printed MD5
 matches a re-computed hash, and the file unpacks correctly. The test cases are:
 

get-lite-file-from-azure.ps1

@@ -7,13 +7,15 @@ param(
     [switch]$Asn
 )
 
-# Base URL for all data files
-$BaseUrl = "https://51ddatafiles.blob.core.windows.net/enterpriseipi/"
+# The Distributor's free downloads, which need no License Key. The source
+# value lets downloads be counted by where they come from.
+$BaseUrl = "https://distributor.51degrees.com/api/v2/free/"
+$Source = "ip-intelligence-data"
 
-# Known files: remote basename + local .ipi name
+# Known files: download name + local .ipi name
 $KnownFiles = @(
-    [pscustomobject]@{ Key = "Lite"; Remote = "51Degrees-IPIV4LiteIpiV41.ipi.gz"; Name = "51Degrees-LiteV41.ipi" }
-    [pscustomobject]@{ Key = "Asn";  Remote = "51Degrees-IPIV4AsnIpiV41.ipi.gz";  Name = "51Degrees-IPIV4AsnIpiV41.ipi" }
+    [pscustomobject]@{ Key = "Lite"; Remote = "ipi-lite-v41"; Name = "51Degrees-LiteV41.ipi" }
+    [pscustomobject]@{ Key = "Asn";  Remote = "ipi-asn-v41";  Name = "51Degrees-IPIV4AsnIpiV41.ipi" }
 )
 
 # Selection: if neither -Lite nor -Asn is passed, do all known files.
@@ -25,27 +27,87 @@ $Selected = @($KnownFiles | Where-Object {
 })
 if (-not $Selected) { $Selected = $KnownFiles }
 
+# Unpacks the gzip data at Start into Out, and returns the length unpacked
+# as a gzip member records it, modulo 2^32.
+function Copy-Gzip([string]$Path, [long]$Start, $Out) {
+    $before = $Out.Position
+    $in = [File]::OpenRead($Path)
+    try {
+        $in.Position = $Start
+        $gzip = [Compression.GZipStream]::new($in, [Compression.CompressionMode]::Decompress)
+        try { $gzip.CopyTo($Out) } finally { $gzip.Dispose() }
+    } finally { $in.Dispose() }
+    [uint32](($Out.Position - $before) % 4294967296)
+}
+
+# The length the last gzip member records, in the last 4 bytes of the file.
+function Get-LastMemberSize([string]$Path) {
+    $in = [File]::OpenRead($Path)
+    try {
+        $tail = New-Object byte[] 4
+        $in.Position = $in.Length - 4
+        [void]$in.Read($tail, 0, 4)
+        [BitConverter]::ToUInt32($tail, 0)
+    } finally { $in.Dispose() }
+}
+
+# Where the gzip member after one that unpacked to Size bytes starts, found
+# as that length followed by the 1f 8b 08 that begins the next member, or
+# -1 when there is none.
+function Find-NextMember([string]$Path, [long]$From, [uint32]$Size) {
+    [byte[]]$pattern = [BitConverter]::GetBytes($Size) + [byte[]](0x1f, 0x8b, 0x08)
+    $in = [File]::OpenRead($Path)
+    try {
+        $buffer = New-Object byte[] (1MB)
+        $position = $From
+        while ($position -lt $in.Length) {
+            $in.Position = $position
+            $count = $in.Read($buffer, 0, $buffer.Length)
+            if ($count -lt $pattern.Length) { break }
+            $i = [Array]::IndexOf($buffer, $pattern[0], 0, $count)
+            while ($i -ge 0 -and $i -le $count - $pattern.Length) {
+                $match = $true
+                for ($j = 1; $j -lt $pattern.Length; $j++) {
+                    if ($buffer[$i + $j] -ne $pattern[$j]) { $match = $false; break }
+                }
+                if ($match) { return $position + $i + 4 }
+                $i = [Array]::IndexOf($buffer, $pattern[0], $i + 1, $count - $i - 1)
+            }
+            $position += $count - ($pattern.Length - 1)
+        }
+    } finally { $in.Dispose() }
+    -1
+}
+
+# The Distributor sends each file as more than one gzip member. PowerShell 7
+# unpacks them all in one pass, but Windows PowerShell stops at the end of
+# the first, so there each member is unpacked in turn.
 function Expand-GzipFile {
     param(
         [Parameter(Mandatory)] [string]$ArchiveName,
         [Parameter(Mandatory)] [string]$ArchivedName
     )
     Write-Host "Extracting '$ArchiveName' to '$ArchivedName'..."
+    $out = [File]::Create($ArchivedName)
     try {
-        $src = [File]::OpenRead($ArchiveName)
-        $dest = [File]::Create($ArchivedName)
-        $gunzip = [Compression.GZipStream]::new($src, [Compression.CompressionMode]::Decompress)
-        $gunzip.copyTo($dest)
-    } finally {
-        # Avoid calling Close on nulls
-        if ($gunzip) { $gunzip.Close() }
-        if ($dest) { $dest.Close() }
-        if ($src) { $src.Close() }
-    }
+        if ($PSVersionTable.PSEdition -eq 'Core') {
+            $null = Copy-Gzip $ArchiveName 0 $out
+            return
+        }
+        $lastSize = Get-LastMemberSize $ArchiveName
+        $start = 0L
+        while ($true) {
+            $size = Copy-Gzip $ArchiveName $start $out
+            if ($size -eq $lastSize) { return }
+            # A member is at least 20 bytes, its length in the last 4.
+            $start = Find-NextMember $ArchiveName ($start + 16) $size
+            if ($start -lt 0) { throw "'$ArchiveName' did not unpack." }
+        }
+    } finally { $out.Dispose() }
 }
 
 foreach ($file in $Selected) {
-    $ArchiveUrl = $BaseUrl + $file.Remote
+    $ArchiveUrl = "$BaseUrl$($file.Remote)?source=$Source"
     $ArchivedName = (Join-Path (Get-Location) $file.Name)
     $ArchiveName = "$ArchivedName.gz"
 
@@ -63,4 +125,4 @@ foreach ($file in $Selected) {
     # Unpack
     Write-Output "Extracting $ArchiveName"
     Expand-GzipFile -ArchiveName $ArchiveName -ArchivedName $ArchivedName
-}
+}

get-lite-file-from-azure.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
 
-# Base URL for all data files
-BASE_URL="https://51ddatafiles.blob.core.windows.net/enterpriseipi/"
+# The Distributor's free downloads, which need no License Key. The source
+# value lets downloads be counted by where they come from.
+BASE_URL="https://distributor.51degrees.com/api/v2/free/"
+SOURCE="ip-intelligence-data"
 
-# Known files: "remote basename|local .ipi name"
-LITE_FILE="51Degrees-IPIV4LiteIpiV41.ipi.gz|51Degrees-LiteV41.ipi"
-ASN_FILE="51Degrees-IPIV4AsnIpiV41.ipi.gz|51Degrees-IPIV4AsnIpiV41.ipi"
+# Known files: "download name|local .ipi name"
+LITE_FILE="ipi-lite-v41|51Degrees-LiteV41.ipi"
+ASN_FILE="ipi-asn-v41|51Degrees-IPIV4AsnIpiV41.ipi"
 
 # Default values
 FORCE=false
@@ -48,12 +50,13 @@ fi
 process_file() {
     local remote="$1"
     local archived_name="$2"
-    local archive_url="${BASE_URL}${remote}"
+    local archive_url="${BASE_URL}${remote}?source=${SOURCE}"
     local archive_name="${archived_name}.gz"
 
     # Download if forced or archive doesn't exist
     if [ "$FORCE" = true ] || [ ! -f "$archive_name" ]; then
-        curl -o "$archive_name" "$archive_url"
+        # -f fails on an error answer instead of saving it as the archive.
+        curl -fL -o "$archive_name" "$archive_url"
     else
         echo "Archive found. Download skipped."
     fi

tests/check-dl-script.tests.ps1

@@ -2,7 +2,7 @@
 # Pester test suite for get-lite-file-from-azure.ps1/.sh download scripts.
 #
 # Data-driven tests: one case per (script x invocation mode). Each case
-# performs a REAL download from Azure blob storage into an isolated temp dir.
+# performs a REAL download from the Distributor into an isolated temp dir.
 
 [CmdletBinding()]
 param()