; This procedure reads ozone lidar data written in the TOLNet-compliant format
; For questions/comments, contact Thierry Leblanc at leblanc@tmf.jpl.nasa.gov
; but... I will not answer your questions if you do not know IDL programming

pro ReadTOLNet,file ,instr_name,pi_name,pi_org,pi_email,site_name,site_lon,site_lat,site_alt,rev_id $                       ; Returned parameters for general header
                    ,process_date,process_time,process_v,qual,date_start,time_start,date_end,time_end,date_mean,time_mean $ ; Returned parameters for profile header
                    ,xsce_name,xsce_date,xsce_time,xsce_lon,xsce_lat,xsce_alt,oper_comm   $                                 ; Returned parameters for profile header (continued)
                    ,z,o3nd,uo3nd,zres,prec,chancode,o3mr,uo3mr,p,up,t,ut,d,ud $                                            ; Returned parameters for data columns
                    ,rev_comm=rev_comm,prof_comm=prof_comm,ndata=ndata,extract=extract                                      ; Retuned keyword parameters (optional)

; Returned parameters' and keywords' dimension and type:
;-------------------------------------------------------------------------------------------------------------
; If only 1 profile is extracted:
; integer scalar:  rev_id
; string scalar:   file,instr_name, pi_name, pi_org, pi_email, site_name, all dates and times, process_v, qual, xsce_name, all xsce_dates and times, oper_comm
; floating scalar: site_lon, site_lat, site_ alt, xsce_lon, xcsce_lat, xsce_alt
; floating vector of length equal to the number of sampled altitudes (ndata): z, o3nd, uo3nd, zres, prec, chancode, o3mr, uo3mr, p, up, t, ut, d, ud
; string vector of length equal to the number of optional comment lines: rev_comm, prof_comm
;-------------------------------------------------------------------------------------------------------------
; If more than 1 profile is extracted:
; The "general header" returned variables have their dimension unchanged
; All "profile header" returned variables have their dimension augmented by one (added in front), which typically are vectors of length "nprof" instead of scalars
; All returned "data" variables have their dimension augmented by one (added in front), which typically are arrays of dimension "nprof x ndata" instead of vectors of length "ndata"
; Note: Use the keyword syntax ",ndata=[..,..,..]" (a vector of length "nprof") to return the number of data lines for each profile
; Note: Use the keyword syntax ",extract=[..,..,..]" to return a subset of the profiles (profiles # starting at 1)

; Calling options:
;-------------------------------------------------------------------------------------------------------------
; To read a data file and extract all its profiles, use the following syntax:
;
; ReadTOLNet,file ,instr_name,pi_name,pi_org,pi_email,site_name,site_lon,site_lat,site_alt,rev_id  $
;                 ,process_date,process_time,process_v,qual,date_start,time_start,date_end,time_end,date_mean,time_mean $
;                 ,xsce_name,xsce_date,xsce_time,xsce_lon,xsce_lat,xsce_alt,oper_comm $
;                 ,z,o3nd,uo3nd,zres,prec,chancode,o3mr,uo3mr,p,up,t,ut,d,ud [,prof_comm=...][,rev_comm=...][,ndata=...]

;-------------------------------------------------------------------------------------------------------------
; To read a data file and extract a subset of its profiles (subset speficied by a vector of profile #), use the following syntax:
;
; ReadTOLNet,file,extract=[..,..] ,instr_name,pi_name,pi_org,pi_email,site_name,site_lon,site_lat,site_alt,rev_id  $
;                                 ,process_date,process_time,process_v,qual,date_start,time_start,date_end,time_end,date_mean,time_mean $
;                                 ,xsce_name,xsce_date,xsce_time,xsce_lon,xsce_lat,xsce_alt,oper_comm $
;                                 ,z,o3nd,uo3nd,zres,prec,chancode,o3mr,uo3mr,p,up,t,ut,d,ud [,prof_comm=...][,rev_comm=...][,ndata=...]

;-------------------------------------------------------------------------------------------------------------
; Optional keywords:
; To read and return the revision comment lines, use the optional kewyord syntax "rev_comm=..."
; To read and return the profile comment lines, use the op[tional keyword syntax "prof_comm=..."
; To read and return the number of dtaa lines for each profile, use the optional keyword syntax "ndata=..."
;-------------------------------------------------------------------------------------------------------------

; Programming history
;-------------------------------------------------------------------------------------------------------------
; 2012-12-03: Version 1.00
;-------------------------------------------------------------------------------------------------------------

a=''
; 1) Open/close file for quick access to key info before actual data reading
;-------------------------------------------------------------------------------------------------------------
openr,lun,/get_lun,file
  readf,lun,ngenhead
  readf,lun,a & b=strtrim(strsplit(a,';',/regex,/extract),2) & formatv=b(0) & if formatv eq 'v1.0' then begin & ngenformat=5 & ngenmeta=5 & nrecmeta=10 & nrecformat=3 & endif
  readf,lun,nprof & if keyword_set(extract) then extract=extract else extract=1+indgen(nprof) & ndata=intarr(nprof) & nprofcomm=intarr(nprof)
  readf,lun,ndatacol
  skip_lun,lun,ndatacol+1,/lines
  readf,lun,ngencomm & nrevcomm=ngencomm-ngenmeta
  skip_lun,lun,ngencomm,/lines
  nprofcommax=0 & ndatamax=0
  for iprof=0,nprof-1 do begin
    igood=where(extract eq iprof+1,ngood)
    if ngood eq 0 then begin
      skip_lun,lun,1,/line
      readf,lun,nrecr & readf,lun,ndatr
      skip_lun,lun,nrecr-1,/line & skip_lun,lun,ndatr,/line
    endif else begin
      skip_lun,lun,1,/lines
      readf,lun,nrecr & nprofcomm(iprof)=nrecr-nrecformat+1-nrecmeta & nprofcommax=nprofcommax>nprofcomm(iprof)
      readf,lun,ndatr & ndata(iprof)=ndatr                           & ndatamax=ndatamax>ndata(iprof)
      skip_lun,lun,nrecr-1,/lines
      skip_lun,lun,ndatr,/lines
    endelse
  endfor
close,lun & free_lun,lun

openr,lun,/get_lun,file

; 2) Read general header
;-------------------------------------------------------------------------------------------------------------
readf,lun,ngenhead
readf,lun,formatv
readf,lun,nprof &
readf,lun,ndatacol & varname=strarr(ndatacol) & varcolumn=intarr(ndatacol) & missval=fltarr(ndatacol)
for idatacol=0,ndatacol-1 do begin
  readf,lun,a & b=strsplit(a,';',/regex,/extract) & c=strsplit(b(0),',',/regex,/extract) & varname(idatacol)=strtrim(c(0),2) & varcolumn(idatacol)=fix ( strmid( c(1),8,strlen(c(1))-8))
endfor
readf,lun,a & b=strtrim(strsplit(a,';',/regex,/extract),2) & c=strsplit(b(0),', ',/regex,/extract) & missval=float(c)

; 3) Read general meta data and comments
;-------------------------------------------------------------------------------------------------------------
readf,lun,ngencomm & gencomm=strarr(ngencomm+1) & labelcomm=gencomm & if nrevcomm gt 0 then rev_comm=strarr(nrevcomm)
for igencomm=1,ngencomm do begin
  readf,lun,a & b=strtrim(strsplit(a,';',/regex,/extract),2) & gencomm(igencomm)=b(0) & labelcomm(igencomm)=b(1)
endfor
instr_name=gencomm(min(where(labelcomm        eq 'INSTRUMENT NAME')))
pi_info=gencomm(where(labelcomm           eq 'PI AND CONTACT INFO')) & c=strtrim(strsplit(pi_info,',',/regex,/extract),2) & pi_name=c(0) & pi_org=c(1) & pi_email=c(2)
site_name=gencomm(min(where(labelcomm         eq 'SITE NAME')))
geoloc1=gencomm(where(labelcomm           eq 'SITE LONGITUDE, LATITUDE, ELEVATION (degE, degN, m)')) & c=strtrim(strsplit(geoloc1,',',/regex,/extract),2) & site_lon=c(0) & site_lat=c(1) & site_alt=c(2)
rev_id=fix(strmid(gencomm(min(where(labelcomm eq 'DATA REVISION # (if value >0 then provide text below)'))),1,4))
if nrevcomm gt 0 then for irevcomm=0,nrevcomm-1 do rev_comm(irevcomm)=gencomm(ngencomm-nrevcomm+1+irevcomm)

; 4) Assign profile header meta data and data dimensions
;-------------------------------------------------------------------------------------------------------------
ndata=intarr(nprof) & qual=strarr(nprof) & xsce_lon=fltarr(nprof) & z=dblarr(nprof,ndatamax) & if nprofcommax gt 0 then prof_comm=strarr(nprof,nprofcommax)
process_date=qual & process_time=qual & process_v=qual
date_start=qual & date_end=qual & date_mean=qual & time_start=qual & time_end=qual & time_mean=qual
xsce_name=qual & xsce_date=qual & xsce_time=qual & xsce_lat=xsce_lon & xsce_alt=xsce_lon & oper_comm=qual
o3nd=z & uo3nd=z & zres=z & prec=z & chancode=z & o3mr=z & uo3mr=z & p=z & up=z & t=z & ut=z & d=z & ud=z


; 5) Loop over profiles
;-------------------------------------------------------------------------------------------------------------
for iprof=0,nprof-1 do begin

; Test wether profile should be extracted or not
  iextract=where(extract eq iprof+1,nextract)

  if nextract eq 0 then begin

; Skip profile (not to be extracted)
    skip_lun,lun,1,/line
    readf,lun,nrecr & readf,lun,ndatr
    skip_lun,lun,nrecr-1,/line & skip_lun,lun,ndatr,/line

  endif else begin

; Read profile header and comments
    skip_lun,lun,1,/line    ; profile separator
    readf,lun,nrechead & rechead=strarr(nrechead+1) & labelhead=rechead
    for irechead=1,nrechead-1 do begin
      readf,lun,a & b=strtrim(strsplit(a,';',/regex,/extract),2) & rechead(irechead)=b(0) & labelhead(irechead)=b(1)
    endfor
    ndata(iprof)=rechead(where(labelhead     eq 'NUMBER OF DATA LINES IN THIS PROFILE'))
    datetime1=rechead(where(labelhead        eq 'DATA PROCESSING DATE, TIME')) & c=strtrim(strsplit(datetime1,',',/regex,/extract),2) & process_date(iprof)=c(0) & process_time(iprof)=c(1)
    process_v(iprof)=rechead(where(labelhead eq 'DATA PROCESSING VERSION'))
    qual(iprof)=rechead(where(labelhead      eq 'RESULTS QUALITY (NOMINAL, FAIR, POOR)'))
    datetime2=rechead(where(labelhead        eq 'PROFILE DATE, TIME (UT) START')) & c=strtrim(strsplit(datetime2,',',/regex,/extract),2) & date_start(iprof)=c(0) & time_start(iprof)=c(1)
    datetime3=rechead(where(labelhead        eq 'PROFILE DATE, TIME (UT) END'))   & c=strtrim(strsplit(datetime3,',',/regex,/extract),2) & date_end(iprof)=c(0)   & time_end(iprof)=c(1)
    datetime4=rechead(where(labelhead        eq 'PROFILE DATE, TIME (UT) START')) & c=strtrim(strsplit(datetime4,',',/regex,/extract),2) & date_mean(iprof)=c(0)  & time_mean(iprof)=c(1)
    xsce_name(iprof)=rechead(where(labelhead eq 'SOURCE OF A PRIORI Press, Temp, AirND USED TO DERIVE OZONE MIXING RATIO'))
    datetime5=rechead(where(labelhead        eq 'SOURCE DATE, TIME (UT)')) & c=strtrim(strsplit(datetime5,',',/regex,/extract),2) & xsce_date(iprof)=c(0) & xsce_time(iprof)=c(1)
    geoloc2=rechead(where(labelhead          eq 'SOURCE LONGITUDE, LATITUDE, ELEVATION (degE, degN, m)')) & c=strtrim(strsplit(geoloc2,',',/regex,/extract),2) & xsce_lon(iprof)=c(0) & xsce_lat(iprof)=c(1) & xsce_alt(iprof)=c(2)
    oper_comm(iprof)=rechead(where(labelhead eq 'OPERATOR COMMENTS'))
    if nprofcomm(iprof) gt 0 then for iprofcomm=0,nprofcomm(iprof)-1 do prof_comm(iprof,iprofcomm)=rechead(nrechead-nprofcomm(iprof)+iprofcomm)
    readf,lun,a & b=strtrim(strsplit(a,';',/regex,/extract),2) & columnlabels=strtrim(strsplit(b(0),',',/regex,/extract),2) & for idatacol=0,ndatacol-1 do if columnlabels(idatacol) ne varname(idatacol) then message,'Inconsistent variable names'

; Read profile data
    datar=dblarr(ndatacol,ndata(iprof))
    readf,lun,datar
    icolgood=where(columnlabels eq 'ALT')         & z(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'O3ND')        & o3nd(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'O3NDUncert')  & uo3nd(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'O3NDResol')   & zres(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'Precision')   & prec(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'ChRange')     & chancode(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'O3MR')        & o3mr(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'O3MRUncert')  & uo3mr(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'Press')       & p(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'PressUncert') & up(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'Temp')        & t(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'TempUncert')  & ut(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'AirND')       & d(iprof,0:ndata(iprof)-1)=datar(icolgood,*)
    icolgood=where(columnlabels eq 'AirNDUncert') & ud(iprof,0:ndata(iprof)-1)=datar(icolgood,*)

  endelse

endfor

close,lun & free_lun,lun

; Reduce returned arrays' size based on requested subset
if n_elements(extract) lt nprof then begin
  process_date=process_date([extract-1])
  process_time=process_time([extract]-1) & process_v=process_v([extract]-1) & qual=qual([extract]-1)
  date_start=date_start([extract]-1) & time_start=time_start([extract]-1) & date_end=date_end([extract]-1)
  time_end=time_end([extract]-1) & date_mean=date_mean([extract]-1) & time_mean=time_mean([extract]-1)
  xsce_name=xsce_name([extract]-1) & xsce_date=xsce_date([extract]-1) & xsce_time=xsce_time([extract]-1) & xsce_lon=xsce_lon([extract]-1) & xsce_lat=xsce_lat([extract]-1) & xsce_alt=xsce_alt([extract]-1)
  oper_comm=oper_comm([extract]-1) & ndata=ndata([extract]-1)
  z=z([extract]-1,*) & o3nd=o3nd([extract]-1,*) & uo3nd=uo3nd([extract]-1,*) & zres=zres([extract]-1,*) & prec=prec([extract]-1,*) & chancode=chancode([extract]-1,*)
  o3mr=o3mr([extract]-1,*) & uo3mr=uo3mr([extract]-1,*) & p=p([extract]-1,*) & up=up([extract]-1,*) & t=t([extract]-1,*) & ut=ut([extract]-1,*) & d=d([extract]-1,*) & ud=ud([extract]-1,*)
  if nprofcommax gt 0 then prof_comm=prof_comm([extract]-1,*)
endif

; Minimize returned arrays' dimensions to scalars and vectors if only 1 profile is returned
if n_elements(extract) eq 1 then begin
  process_date=process_date(0) & process_time=process_time(0) & process_v=process_v(0) & qual=qual(0)
  date_start=date_start(0) & time_start=time_start(0) & date_end=date_end(0)
  time_end=time_end(0) & date_mean=date_mean(0) & time_mean=time_mean(0)
  xsce_name=xsce_name(0) & xsce_date=xsce_date(0) & xsce_time=xsce_time(0) & xsce_lon=xsce_lon(0) & xsce_lat=xsce_lat(0) & xsce_alt=xsce_alt(0)
  oper_comm=oper_comm(0) & ndata=ndata(0)
  z=reform(z) & o3nd=reform(o3nd) & uo3nd=reform(uo3nd) & zres=reform(zres) & prec=reform(prec) & chancode=reform(chancode)
  o3mr=reform(o3mr) & uo3mr=reform(uo3mr) & p=reform(p) & up=reform(up) & t=reform(t) & ut=reform(ut) & d=reform(d) & ud=reform(ud)
  if nrevcomm gt 0 then rev_comm=reform(rev_comm) & if nprofcommax gt 0 then prof_comm=reform(prof_comm)
endif

return
end