# wdc_kp_ap.jyds
#
# Reads data from the World Data Center in a format that is beyond the specification of the AsciiTableParser.
# Example file: ftp://ftp.ngdc.noaa.gov/STP/GEOMAGNETIC_DATA/INDICES/KP_AP/1942

# getAp=1 read Ap, otherwise read Kp.
# see apkp. getAp= getParam( 'getAp', 0 )

apkp= getParam( 'apkp', 'kp', 'read Ap or Kp', ['ap','kp' ] )
if ( apkp=='ap' ):
  getAp= 1
else:
  getAp= 0

resourceURI= getParam( 'resourceURI', 'ftp://ftp.ngdc.noaa.gov/STP/GEOMAGNETIC_DATA/INDICES/KP_AP/1942', 'example file to load' )

# Download the file to make it local.  Monitor is a progress monitor for the download.
afile= getFile( resourceURI, monitor ).toString()

# Get the year from the last four characters of the resource name.
syear= afile[-4:]

fp= open( afile, "r" )

# Import the DataSetBuilder class which is used to build up the dataset.
from org.virbo.dsutil import DataSetBuilder

timeparser= TimeParser.create( '$Y$m$d' )
timetags= DataSetBuilder( 1, 100 )   # create builders for the data and timetags, allocate 100 records initially.
builder= DataSetBuilder( 1, 100 )

#Here is an example line, the first record from the 1942 file:
#42 1 1148718 3 71010 3101010 63  2  3  4  4  2  4  4  4  30.10 46  0.03
if ( getAp ):
   offs= 31
   leng= 3
   builder.putProperty( QDataSet.NAME, 'Ap' )
   builder.putProperty( QDataSet.TITLE, 'Ap' )
else:
   offs= 12
   leng= 2
   builder.putProperty( QDataSet.NAME, 'Kp' )
   builder.putProperty( QDataSet.TITLE, 'Kp' )

for line in fp:
    stime= syear + line[2:6]     # form YYYYMMDD by adding the year to MMDD in the 2,3,4,and 5th characters
    base_t2000= timeparser.parse( stime ).getTime( Units.t2000 )   # parse the time and return it in seconds since 2000-01-01T00:00Z.
    for j in xrange(8):          # for j=0,1,2,3,4,5,6,7
       i= offs + j*leng
       d= int( line[i:i+leng] )
       timetags.putValue( -1, base_t2000 + 3600 * ( 1.5 + j*3 ) )  # add the offset seconds for each of 8 measurements.
       builder.putValue( -1, d )
    timetags.nextRecord()
    builder.nextRecord()

fp.close()

timetags.putProperty( QDataSet.UNITS, Units.t2000 )                # indicate the times are seconds since 2000-01-01T00:00.
builder.putProperty( QDataSet.DEPEND_0, timetags.getDataSet() )    # connect the timetags to the data.

data= builder.getDataSet()                                         # data is the resultant dataset.