<p>The URL strings are base64-encoded, the example given decodes to:
<code>/JRA3Q/Client9120/Daily/anl_isentrop/anl_isentrop_bvf2.cglt9103.day0107</code></p>
<p>With this information, you can pretty easily figure out, for a given dataset, which parts of the string you need to replace.  Then, use the provided download script:</p>
<pre><code>#!/usr/bin/env python

import urllib
import urllib2
import urlparse
import cookielib
import HTMLParser
import subprocess
import sys
import os


import optparse
import netrc
import getpass


class CASLoginParser(HTMLParser.HTMLParser):
    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.action = None
        self.data = {}

    def handle_starttag(self, tagname, attribute):
        if tagname.lower() == 'form':
            attribute = dict(attribute)
            if 'action' in attribute:
                self.action = attribute['action']
        elif tagname.lower() == 'input':
            attribute = dict(attribute)
            if 'name' in attribute and 'value' in attribute:
                self.data[attribute['name']] = attribute['value']

class DIASAccess():
    def __init__(self, username, password):
        self.__cas_url = 'https://auth.diasjp.net/cas/login?'
        self.__username = username
        self.__password = password
        #self.__cj = cookielib.CookieJar()
        self.__cj = cookielib.MozillaCookieJar()
        self.__opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self.__cj))

    def open(self, url, data=None):
        response = self.__opener.open(url, data)
        response_url = response.geturl()

        if response_url != url and response_url.startswith(self.__cas_url):
            # redirected to CAS login page
            response = self.__login_cas(response)
            if data != None:
                # If POST (data != None), need reopen
                response.close()
                response = self.__opener.open(url, data)

        return response

    def __login_cas(self, response):
        parser = CASLoginParser()
        parser.feed(response.read())
        parser.close()

        if parser.action == None:
            raise LoginError('Not login page')

        action_url = urlparse.urljoin(response.geturl(), parser.action)
        data = parser.data
        data['username'] = self.__username
        data['password'] = self.__password

        response.close()
        response = self.__opener.open(action_url, urllib.urlencode(data))

        if response.geturl() == action_url:
            print 'Authorization fail'
            quit()

        return response

    def dl(self, url, path, file, data=None):
        try:
            response = self.__opener.open(url, data)
            if not os.path.exists('.' + path):
                os.makedirs('.' + path)

            f = open('.' + path + file, 'wb')
            file_size_dl = 0
            block_size = 8192
            while True:
                buffer = response.read(block_size)
                if not buffer:
                    break

                file_size_dl += len(buffer)
                f.write(buffer)

            f.close
            print path + file + &quot;  OK&quot;
            return response

        except urllib2.HTTPError,e:
            print path + file + &quot;  NG&quot;


class LoginError(Exception):
    def __init__(self, e):
        Exception.__init__(self, e)

if __name__ == '__main__':
    host = 'data.diasjp.net'

    usage ='''usage: %prog [options]'''
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('-n', '--netrc', default=None,
                      help='specify the netrc file', metavar='FILE')
    parser.add_option('-u', '--user', default=None,
                      help='specify the DIAS account name',
                      metavar='USERNAME')

    (options, args) = parser.parse_args()

    (login, password) = (None, None)

    try:
      auth = netrc.netrc(options.netrc).authenticators(host)
      if auth is not None:
            (login, account, password) = auth
    except (IOError):
      pass

    if options.user is not None:
        login = options.user
        password = None

    if login is None:
        login = raw_input('Username: ')

    if password is None:
        password = getpass.getpass('Password: ')

    access = DIASAccess(login, password)


    targeturl='https://data.diasjp.net/dl/storages/filelist/dataset:645'
    response = access.open(targeturl)
    response.close()


    access.dl('https://data.diasjp.net/dl/storages/downloadCmd/L0pSQTNRL0NsaW05MTIwL0RhaWx5L2FubF9pc2VudHJvcC9hbmxfaXNlbnRyb3BfYnZmMi5jbGltOTEyMC5kYXkwMTAx', '/JRA3Q/Clim9120/Daily/anl_isentrop/', 'anl_isentrop_bvf2.clim9120.day0101')
</code></pre>
<p>and replace the parameters in <code>access.dl</code> appropriately.</p>

CC BY-SA 4.0
2023-08-16T01:57:53.283Z
Great work ! I was searching for something similar online. Can you actually download all the variables - zonal wind, meridional wind etc ? Do you have a Jupyter notebook on a github site possibly that I can run ?
gansub
CC BY-SA 4.0
2023-08-16T10:39:37.603Z
If you can share me your github site I can suggest enhancements to this code by allowing for pressure level downloads, surface level parameters etc and we can do a joint release.
gansub
CC BY-SA 4.0
2023-08-15T21:12:33.807Z
2023-08-15T21:12:33.807Z
alex_danielssen
<p>The JRA-3Q dataset is available from <a href="https://search.diasjp.net/en/dataset/JRA3Q" rel="nofollow noreferrer">DIAS</a>. There is an online interface to download what you need file-by-file.</p>
<p>Other reanalyses like ERA5 and MERRA2 have access points that let you write a python script to get what you want (and often subset it on their servers using opendap or similar).</p>
<p>I haven't been able to find an efficient way of downloading the JRA-3Q dataset in such a way.  Part of this is the file nomenclature that includes an obscure-looking string (e.g.</p>
<p><code>https://data.diasjp.net/dl/storages/file/L0pSQTNRL0NsaW05MTIwL0RhaWx5L2FubF9pc2VudHJvcC9hbmxfaXNlbnRyb3BfYnZmMi5jbGltOTEyMC5kYXkwMTA3</code></p>
<p>Has anyone come up with a way to download quite a lot of data in an efficient way?</p>

<climate><open-data><python><reanalysis><climate-data>
Downloading JRA-3Q efficiently using a python script
alex_danielssen
# Downloading JRA-3Q efficiently using a python script

(Asked by: username_0 on 2023-08-15)

The JRA-3Q dataset is available from [DIAS](https://search.diasjp.net/en/dataset/JRA3Q). There is an online interface to download what you need file-by-file.

Other reanalyses like ERA5 and MERRA2 have access points that let you write a python script to get what you want (and often subset it on their servers using opendap or similar).

I haven't been able to find an efficient way of downloading the JRA-3Q dataset in such a way. Part of this is the file nomenclature that includes an obscure-looking string (e.g.

`https://data.diasjp.net/dl/storages/file/L0pSQTNRL0NsaW05MTIwL0RhaWx5L2FubF9pc2VudHJvcC9hbmxfaXNlbnRyb3BfYnZmMi5jbGltOTEyMC5kYXkwMTA3`

Has anyone come up with a way to download quite a lot of data in an efficient way?

**Tags:** climate, open-data, python, reanalysis, climate-data


---

## 1 answers

### Answer by username_0 on 2023-08-15. Score: 3

The URL strings are base64-encoded, the example given decodes to:
`/JRA3Q/Client9120/Daily/anl_isentrop/anl_isentrop_bvf2.cglt9103.day0107`

With this information, you can pretty easily figure out, for a given dataset, which parts of the string you need to replace. Then, use the provided download script:

```
#!/usr/bin/env python

import urllib
import urllib2
import urlparse
import cookielib
import HTMLParser
import subprocess
import sys
import os

import optparse
import netrc
import getpass

class CASLoginParser(HTMLParser.HTMLParser):
    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.action = None
        self.data = {}

    def handle_starttag(self, tagname, attribute):
        if tagname.lower() == 'form':
            attribute = dict(attribute)
            if 'action' in attribute:
                self.action = attribute['action']
        elif tagname.lower() == 'input':
            attribute = dict(attribute)
            if 'name' in attribute and 'value' in attribute:
                self.data[attribute['name']] = attribute['value']

class DIASAccess():
    def __init__(self, username, password):
        self.__cas_url = 'https://auth.diasjp.net/cas/login?'
        self.__username = username
        self.__password = password
        #self.__cj = cookielib.CookieJar()
        self.__cj = cookielib.MozillaCookieJar()
        self.__opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self.__cj))

    def open(self, url, data=None):
        response = self.__opener.open(url, data)
        response_url = response.geturl()

        if response_url != url and response_url.startswith(self.__cas_url):
            # redirected to CAS login page
            response = self.__login_cas(response)
            if data != None:
                # If POST (data != None), need reopen
                response.close()
                response = self.__opener.open(url, data)

        return response

    def __login_cas(self, response):
        parser = CASLoginParser()
        parser.feed(response.read())
        parser.close()

        if parser.action == None:
            raise LoginError('Not login page')

        action_url = urlparse.urljoin(response.geturl(), parser.action)
        data = parser.data
        data['username'] = self.__username
        data['password'] = self.__password

        response.close()
        response = self.__opener.open(action_url, urllib.urlencode(data))

        if response.geturl() == action_url:
            print 'Authorization fail'
            quit()

        return response

    def dl(self, url, path, file, data=None):
        try:
            response = self.__opener.open(url, data)
            if not os.path.exists('.' + path):
                os.makedirs('.' + path)

            f = open('.' + path + file, 'wb')
            file_size_dl = 0
            block_size = 8192
            while True:
                buffer = response.read(block_size)
                if not buffer:
                    break

                file_size_dl += len(buffer)
                f.write(buffer)

            f.close
            print path + file + "  OK"
            return response

        except urllib2.HTTPError,e:
            print path + file + "  NG"

class LoginError(Exception):
    def __init__(self, e):
        Exception.__init__(self, e)

if __name__ == '__main__':
    host = 'data.diasjp.net'

    usage ='''usage: %prog [options]'''
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('-n', '--netrc', default=None,
                      help='specify the netrc file', metavar='FILE')
    parser.add_option('-u', '--user', default=None,
                      help='specify the DIAS account name',
                      metavar='USERNAME')

    (options, args) = parser.parse_args()

    (login, password) = (None, None)

    try:
      auth = netrc.netrc(options.netrc).authenticators(host)
      if auth is not None:
            (login, account, password) = auth
    except (IOError):
      pass

    if options.user is not None:
        login = options.user
        password = None

    if login is None:
        login = raw_input('Username: ')

    if password is None:
        password = getpass.getpass('Password: ')

    access = DIASAccess(login, password)

    targeturl='https://data.diasjp.net/dl/storages/filelist/dataset:645'
    response = access.open(targeturl)
    response.close()

    access.dl('https://data.diasjp.net/dl/storages/downloadCmd/L0pSQTNRL0NsaW05MTIwL0RhaWx5L2FubF9pc2VudHJvcC9hbmxfaXNlbnRyb3BfYnZmMi5jbGltOTEyMC5kYXkwMTAx', '/JRA3Q/Clim9120/Daily/anl_isentrop/', 'anl_isentrop_bvf2.clim9120.day0101')

```

and replace the parameters in `access.dl` appropriately.

> **Comments:**
> **username_1**: Great work ! I was searching for something similar online. Can you actually download all the variables - zonal wind, meridional wind etc ? Do you have a Jupyter notebook on a github site possibly that I can run ?  
> **username_1**: If you can share me your github site I can suggest enhancements to this code by allowing for pressure level downloads, surface level parameters etc and we can do a joint release.  

---

