diff --git a/README.md b/README.md index 83e4087..238fcea 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This repo allows to obtain data needed to run IP Intelligence examples: ## Download Lite data file The Lite data file is too large to commit to this repository, so one of the -scripts below downloads it from 51Degrees storage instead: +scripts below downloads it from the 51Degrees Distributor instead: | Script | Language | OS | | -- | -- | -- | @@ -66,7 +66,7 @@ This 'lite' file is built from a subsection of all IPs and is updated less frequ ### `get-lite-file-from-azure.ps1`/`.sh` -Convenience scripts to download (and unpack) data file, stored on Azure Blob Storage. +Convenience scripts to download (and unpack) the data files from the 51Degrees Distributor, which needs no License Key. ### `evidence-gen.ps1` @@ -102,7 +102,7 @@ cd ip-intelligence-data ./ci/run-unit-tests.ps1 ``` -Each test performs a **real download** from 51Degrees storage into an isolated +Each test performs a **real download** from the Distributor into an isolated temporary directory and verifies the archive is fetched, its printed MD5 matches a re-computed hash, and the file unpacks correctly. The test cases are: diff --git a/get-lite-file-from-azure.ps1 b/get-lite-file-from-azure.ps1 index 08b6e9c..11e1121 100644 --- a/get-lite-file-from-azure.ps1 +++ b/get-lite-file-from-azure.ps1 @@ -7,13 +7,15 @@ param( [switch]$Asn ) -# Base URL for all data files -$BaseUrl = "https://51ddatafiles.blob.core.windows.net/enterpriseipi/" +# The Distributor's free downloads, which need no License Key. The source +# value lets downloads be counted by where they come from. +$BaseUrl = "https://distributor.51degrees.com/api/v2/free/" +$Source = "ip-intelligence-data" -# Known files: remote basename + local .ipi name +# Known files: download name + local .ipi name $KnownFiles = @( - [pscustomobject]@{ Key = "Lite"; Remote = "51Degrees-IPIV4LiteIpiV41.ipi.gz"; Name = "51Degrees-LiteV41.ipi" } - [pscustomobject]@{ Key = "Asn"; Remote = "51Degrees-IPIV4AsnIpiV41.ipi.gz"; Name = "51Degrees-IPIV4AsnIpiV41.ipi" } + [pscustomobject]@{ Key = "Lite"; Remote = "ipi-lite-v41"; Name = "51Degrees-LiteV41.ipi" } + [pscustomobject]@{ Key = "Asn"; Remote = "ipi-asn-v41"; Name = "51Degrees-IPIV4AsnIpiV41.ipi" } ) # Selection: if neither -Lite nor -Asn is passed, do all known files. @@ -25,27 +27,87 @@ $Selected = @($KnownFiles | Where-Object { }) if (-not $Selected) { $Selected = $KnownFiles } +# Unpacks the gzip data at Start into Out, and returns the length unpacked +# as a gzip member records it, modulo 2^32. +function Copy-Gzip([string]$Path, [long]$Start, $Out) { + $before = $Out.Position + $in = [File]::OpenRead($Path) + try { + $in.Position = $Start + $gzip = [Compression.GZipStream]::new($in, [Compression.CompressionMode]::Decompress) + try { $gzip.CopyTo($Out) } finally { $gzip.Dispose() } + } finally { $in.Dispose() } + [uint32](($Out.Position - $before) % 4294967296) +} + +# The length the last gzip member records, in the last 4 bytes of the file. +function Get-LastMemberSize([string]$Path) { + $in = [File]::OpenRead($Path) + try { + $tail = New-Object byte[] 4 + $in.Position = $in.Length - 4 + [void]$in.Read($tail, 0, 4) + [BitConverter]::ToUInt32($tail, 0) + } finally { $in.Dispose() } +} + +# Where the gzip member after one that unpacked to Size bytes starts, found +# as that length followed by the 1f 8b 08 that begins the next member, or +# -1 when there is none. +function Find-NextMember([string]$Path, [long]$From, [uint32]$Size) { + [byte[]]$pattern = [BitConverter]::GetBytes($Size) + [byte[]](0x1f, 0x8b, 0x08) + $in = [File]::OpenRead($Path) + try { + $buffer = New-Object byte[] (1MB) + $position = $From + while ($position -lt $in.Length) { + $in.Position = $position + $count = $in.Read($buffer, 0, $buffer.Length) + if ($count -lt $pattern.Length) { break } + $i = [Array]::IndexOf($buffer, $pattern[0], 0, $count) + while ($i -ge 0 -and $i -le $count - $pattern.Length) { + $match = $true + for ($j = 1; $j -lt $pattern.Length; $j++) { + if ($buffer[$i + $j] -ne $pattern[$j]) { $match = $false; break } + } + if ($match) { return $position + $i + 4 } + $i = [Array]::IndexOf($buffer, $pattern[0], $i + 1, $count - $i - 1) + } + $position += $count - ($pattern.Length - 1) + } + } finally { $in.Dispose() } + -1 +} + +# The Distributor sends each file as more than one gzip member. PowerShell 7 +# unpacks them all in one pass, but Windows PowerShell stops at the end of +# the first, so there each member is unpacked in turn. function Expand-GzipFile { param( [Parameter(Mandatory)] [string]$ArchiveName, [Parameter(Mandatory)] [string]$ArchivedName ) Write-Host "Extracting '$ArchiveName' to '$ArchivedName'..." + $out = [File]::Create($ArchivedName) try { - $src = [File]::OpenRead($ArchiveName) - $dest = [File]::Create($ArchivedName) - $gunzip = [Compression.GZipStream]::new($src, [Compression.CompressionMode]::Decompress) - $gunzip.copyTo($dest) - } finally { - # Avoid calling Close on nulls - if ($gunzip) { $gunzip.Close() } - if ($dest) { $dest.Close() } - if ($src) { $src.Close() } - } + if ($PSVersionTable.PSEdition -eq 'Core') { + $null = Copy-Gzip $ArchiveName 0 $out + return + } + $lastSize = Get-LastMemberSize $ArchiveName + $start = 0L + while ($true) { + $size = Copy-Gzip $ArchiveName $start $out + if ($size -eq $lastSize) { return } + # A member is at least 20 bytes, its length in the last 4. + $start = Find-NextMember $ArchiveName ($start + 16) $size + if ($start -lt 0) { throw "'$ArchiveName' did not unpack." } + } + } finally { $out.Dispose() } } foreach ($file in $Selected) { - $ArchiveUrl = $BaseUrl + $file.Remote + $ArchiveUrl = "$BaseUrl$($file.Remote)?source=$Source" $ArchivedName = (Join-Path (Get-Location) $file.Name) $ArchiveName = "$ArchivedName.gz" @@ -63,4 +125,4 @@ foreach ($file in $Selected) { # Unpack Write-Output "Extracting $ArchiveName" Expand-GzipFile -ArchiveName $ArchiveName -ArchivedName $ArchivedName -} \ No newline at end of file +} diff --git a/get-lite-file-from-azure.sh b/get-lite-file-from-azure.sh index 03d306f..2d0968f 100644 --- a/get-lite-file-from-azure.sh +++ b/get-lite-file-from-azure.sh @@ -1,11 +1,13 @@ #!/bin/bash -# Base URL for all data files -BASE_URL="https://51ddatafiles.blob.core.windows.net/enterpriseipi/" +# The Distributor's free downloads, which need no License Key. The source +# value lets downloads be counted by where they come from. +BASE_URL="https://distributor.51degrees.com/api/v2/free/" +SOURCE="ip-intelligence-data" -# Known files: "remote basename|local .ipi name" -LITE_FILE="51Degrees-IPIV4LiteIpiV41.ipi.gz|51Degrees-LiteV41.ipi" -ASN_FILE="51Degrees-IPIV4AsnIpiV41.ipi.gz|51Degrees-IPIV4AsnIpiV41.ipi" +# Known files: "download name|local .ipi name" +LITE_FILE="ipi-lite-v41|51Degrees-LiteV41.ipi" +ASN_FILE="ipi-asn-v41|51Degrees-IPIV4AsnIpiV41.ipi" # Default values FORCE=false @@ -48,12 +50,13 @@ fi process_file() { local remote="$1" local archived_name="$2" - local archive_url="${BASE_URL}${remote}" + local archive_url="${BASE_URL}${remote}?source=${SOURCE}" local archive_name="${archived_name}.gz" # Download if forced or archive doesn't exist if [ "$FORCE" = true ] || [ ! -f "$archive_name" ]; then - curl -o "$archive_name" "$archive_url" + # -f fails on an error answer instead of saving it as the archive. + curl -fL -o "$archive_name" "$archive_url" else echo "Archive found. Download skipped." fi diff --git a/tests/check-dl-script.tests.ps1 b/tests/check-dl-script.tests.ps1 index cadd570..953a9f6 100644 --- a/tests/check-dl-script.tests.ps1 +++ b/tests/check-dl-script.tests.ps1 @@ -2,7 +2,7 @@ # Pester test suite for get-lite-file-from-azure.ps1/.sh download scripts. # # Data-driven tests: one case per (script x invocation mode). Each case -# performs a REAL download from Azure blob storage into an isolated temp dir. +# performs a REAL download from the Distributor into an isolated temp dir. [CmdletBinding()] param()