#!/usr/bin/env perl use strict; use warnings; use Getopt::Long qw( :config posix_default bundling no_ignore_case no_auto_abbrev); use JSON; use File::Path qw(make_path); my $source = undef; my $destination = undef; my $token = undef; GetOptions( 's|source=s' => \$source, 'd|destination=s' => \$destination, 't|token=s' => \$token) or die usage(); sub usage { print "Usage:\n"; print " $0 [options]\n\n"; print "Description:\n"; print " This script will recursively download all files if they don't exist\n"; print " from a LAADS URL and stores them to the specified path\n\n"; print "Options:\n"; print " -s|--source [URL] Recursively download files at [URL]\n"; print " -d|--destination [path] Store directory structure to [path]\n"; print " -t|--token [token] Use app token [token] to authenticate\n"; } sub recurse { my $src = $_[0]; my $dest = $_[1]; my $token = $_[2]; print "Recursing $src\n"; my $cmd = "curl -L -b session -s -g --header 'Authorization: Bearer $token' -C - '$src.json'"; my $message = qx{$cmd}; my $listing = decode_json($message); for my $entry (@{$listing->{content}}){ if($entry->{size} == 0){ make_path("$dest/$entry->{name}"); recurse("$src/$entry->{name}", "$dest/$entry->{name}", $token); } } for my $entry (@{$listing->{content}}){ if($entry->{size} != 0 and ! -e "$dest/$entry->{name}"){ print "Downloading $src/$entry->{name} to $dest/$entry->{name}\n"; my $cmd = "curl -L -b session -s -g --header 'Authorization: Bearer $token' -C - '$src/$entry->{name}' > $dest/$entry->{name}"; my $result = system($cmd); die "FAIL : $cmd" unless $result == 0; } else { print "Skipping $entry->{name} ...\n"; } } } if(!defined($source)){ print "Source not set\n"; usage(); die; } if(!defined($destination)){ print "Destination not set\n"; usage(); die; } if(!defined($token)){ print "Token not set\n"; usage(); die } # make sure the destination exists make_path($destination) unless -d $destination; # and download... recurse($source, $destination, $token); 0;