##SCRIPT TO READ MANN DATA AND STORE AS R-TABLES
##ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/README
###MWP PROXIES
## 13 proxies are located at ftp://eclogite.geo.umass.edu/pub/mann/ONLINE-PREPRINTS/Millennium/DATA/PROXIES/
## what is presumably the Morocco proxy is located at ftp://ftp.ngdc.noaa.gov/paleo/treering/chronologies/africa/morc014.crn
## I have stitched these together and stored as text file for loading under this script


#fenno 500,col2   fran010- 988 col2, 

loc<-c("ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/quelc1-o18.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/quelc1-accum.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/quelc2-o18.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/quelc2-accum.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/westgreen-o18.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/tasmania.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/npatagonia.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/fenno-new.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/urals-new.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/itrdb-namer-pc1.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/itrdb-namer-pc2.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/itrdb-namer-pc3.dat",
"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/proxies/fran010.txt")


mwp<-NULL
for (m in 1:7) {
y<-read.table (file=loc[m]);
mwtemp<-ts(y[,2],start=min(y[,1]),end=max(y[,1]))
mwp<-ts.union(mwp,mwtemp)
}
y<-read.fwf(file=loc[8],widths=c(5,8))
mwtemp<-ts(y[,2],start=min(y[,1]),end=max(y[,1]))
mwp<-ts.union(mwp,mwtemp)
for (m in 9:13) {
y<-read.table (file=loc[m]);
mwtemp<-ts(y[,2],start=min(y[,1]),end=max(y[,1]))
mwp<-ts.union(mwp,mwtemp)
}
#NGDC TREE READ FUNCTION
fred<-"ftp://ftp.ngdc.noaa.gov/paleo/treering/chronologies/africa/morc008.crn" #see discussion for identification
widths1<-c(6,4,rep(c(4,3),10));
tree<-read.fwf(file=fred,widths=widths1,skip=3);
start1<-tree[1,2];
n<-nrow(tree);
tree<-tree[1:(n-1),]##last raw has other data
temp<-c(1+2*(1:10));
tree<-tree[temp];
temp<-(tree>9000);
tree[temp]<-NA;
tree<-t(tree);
tree<-c(tree);
temp<-!is.na(tree);
tree<-tree[temp];
end1<-start1+length(tree)-1;
mwtemp<-ts(tree,start=start1,end=end1);
mwp<-ts.union(mwp,mwtemp)

#tidy dimnames
b<-c("V1","V2","V3","V4","V5","V6","V7","V8","V9","V10","V11","V12","V13","V14")
dimnames(mwp)[[2]]<-b
locf<-"c:/climate/data/mann/mwpproxy.tab";
save(mwp,file=locf); ### dim=1521 x 14 proxies 

#DISCONTINUED ALTERNATIVE
#loc<-"http://www.geocities.com/smcintyre25/data/mann/mwpproxy.txt";
###mwpproxy<-scan(file=loc,n=1521*16,skip=1);
###mwpproxy<-matrix(mwpproxy,1521,16,byrow=TRUE);##from 470 to 1990
###mwp<-ts(data=mwpproxy,start=470,end=1990,frequency=1);
###mwp<-ts.union(mwp,tree2);##used to add in Morocco series;
##col1-row; col2 - year; cols 3-15 proxies from eclogite.geo.umass; col 16 - Morocco proxy from ngdc
###locf<-"c:/climate/data/mann/mwpproxy.txt";###scrap
###write.table(mwp,file=locf,quote=FALSE,sep="\t")###scrap

##PROXIES 1400-1994
##these proxies were picked up from Mann FTP site and stitched together 
loc<-"http://www.geocities.com/smcintyre25/data/mann/mannproxy.txt";
proxy<-scan(file=loc,n=113*(1994-1399),skip=3);#reads 67325 items
proxy<-t(array(proxy,dim=c(113,595)));
proxy<-ts(proxy,start=1400,end=1994);#dim(595,113)
proxy<-proxy[,2:113] #chops off year marker
locf<-"c:/climate/data/mann/proxy.tab";
save(proxy,file=locf);

##PROXYLIST
##this includes in the list proxies from Bradley-Jones 1992, Hughes-Diaz 1994, Jones 1998, Crowley-Lowery 2000
#loc<-"http://www.geocities.com/smcintyre25/data/mann/proxylist.txt";
loc<-"c:/climate/data/mann/proxylist.txt"
prname<-read.table(file=loc,sep="\t",header=TRUE,fill=TRUE);
prname<-prname[1:142,];
locf<-"c:/climate/data/mann/prname.tab";
save(prname,file=locf)
#col1- ID; col2 - Mann #; col3- Record; col 4- Country; col5- type; col6- proxy; col7-indicator; col8-reference;
#col9- year1 (mann); col10- year n (Mann); col 11- lat; col 12- long; col13- Mann MWP proxy;
#col14-Bradley-Jones 1992; col15-Hughes-Diaz 1994 Fig 3; col 16-Jones 1998; col17-Crowley-Lowery2000;
#col18- Durbin-Watson; col19-"rho"; col20-trend 1856-1980; col-21 R2 trend 1856-1980; 
#col22- R2 proxy vs MannPC 1902-1980; col 23-DW proxy vs Mann PC 1902-1990; 
#col 24- NGDC.description; col 25 - NGDC.data
#col 26- Lat.estimated; #col27 -Long.Estimated
##note: for reading data assembled in Excel,DO NOT HAVE signs like ',#,- in character sets


##EOF 
##READ AND STORE IN 16 COLUMNS AND 1082 ROWS

loc<-c("ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof01.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof02.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof03.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof04.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof05.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof06.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof07.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof08.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof09.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof10.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof11.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof12.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof13.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof14.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof15.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/eof16.out")

### note: when this is executing, you may have to click OK on Internet prompt

s<-NULL;
for (m in 1:16)  {
skip1<-0;
stemp<-scan(file=loc[m],n=1082*2,skip=skip1);
s<-c(s,stemp);
}
eof<-array(s,dim=c(2,1082*16));##;
eof<-array(eof[2,],dim=c(1082,16));
locf<-"c:/climate/data/mann/eof.tab";
save(eof,file=locf);


###PC
loc<-c("ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc01.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc02.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc03.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc04.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc05.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc06.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc07.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc08.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc09.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc10.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc11.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc12.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc13.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc14.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc15.out",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pc16.out")

s<-NULL;
for (m in 1:16)  {
skip1<-0;
stemp<-scan(file=loc[m],n=92*2,skip=skip1);
s<-c(s,stemp);
}
pc<-array(s,dim=c(2,92*16));##;
pc<-array(pc[2,],dim=c(92,16));
locf<-"c:/climate/data/mann/pc.tab";
save(pc,file=locf);


##GRIDPOINTS - SAVE AS TABLE WITH JONES GRID BOXES
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/gridpoints.loc"
grid2<-read.table(file=loc1) ##the dimension is 1082x5 and is read as a data.frame
##this corresponds to Mann's 1082 locations
#col1 is long index 1,...72 from Greenwich going east; col 2 lat index 1,..36 going N to S; col 3 is grid-box long centre; col 4 is grid-box lat centre;
#the Mann lat index is the sames as the Jones lat index;
#this script puts Jones long index in col 5

for (i in 1:1082) {
latgp<-grid2[i,2];longgp<-grid2[i,1];
if (longgp>36) longgp<-grid2[i,1]-36 else longgp<-grid2[i,1]+36;#converts Mann long-gp to Jones long-gp
grid2[i,5]<-72*(latgp-1)+longgp;###saves series # for reference to Jones data
} # this is inelegant but I couldn't find mod-function in R
save(grid2,file="c:/climate/data/mann/gridpoints.tab")


##EIGENVECTOR WEIGHTS
#col1-eigenvector; col2 - standard deviation; col 3- % of total variance; col4 - cumulative pct of total variance 
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/tpca-eigenvals.out";
lambda<-scan(file=loc1,n=1082*4,skip=0);
lambda<-t(array(lambda,dim=c(4,1082)));
save(lambda,file="c:/climate/data/mann/tpca-eigenvals.tab")


##PCA WEIGHTS
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/pca-weights.out";
pcaweights<-scan(file=loc1,n=128*2,skip=0);
pcaweights<-t(array(pcaweights,dim=c(2,128)));
save(pcaweights,file="c:/climate/data/mann/pcaweights.tab")

###RECONSTRUCTED PC FROM 1400 TO 1980 (VARYING FIRST YEARS)
loc<-c("ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/rpc01.dat",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/rpc02.dat",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/rpc03.dat",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/rpc04.dat",
"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/EIGENVECTORS/rpc05.dat")

rpcyear1<-c(1400,1450,1750,1760,1450)
n<-1981-rpcyear1;
k<-rpcyear1-1399;
rpc<-c(rep(NA,581*5));
rpc<-array(rpc,dim=c(5,581));
dim(rpc);

for (m in 1:5)  {
skip1<-1;
s<-scan(file=loc[m],n=581*2,skip=skip1);
s<-array(s,dim=c(2,n[m]));
rpc[m,k[m]+0:(n[m]-1)]<-s[2,];
}
rpc<-t(rpc);
rpc<-ts(rpc,start=1400,end=1980);##possible conversion to ts format
locf<-"c:/climate/data/mann/rpc.tab";
save(rpc,file=locf);

###NH MEAN 1400-1995
#note: in earlier download this was only to 1995.
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/nhmean.dat";
nhmann<-read.table(loc1,skip=1,fill=TRUE)
b<-c("year","NH.RECON","NH.RAW","-1.sigma","-2.sigma","+1.sigma","+2.sigma")
dimnames(nhmann)[[2]]<-b
##col1-year; col2- NH reconstruction; col3-NH -raw; col4- minus 1 sigma; col5- minus 2 sigma; col6- plus 1 sigma; col7 -plus 2 sigma
nhmann[582:599,c(2,4,5,6,7)]<-NA
nhmann[1:502,3]<-NA
locf<-"c:/climate/data/mann/nhmann.tab";
save(nhmann,file=locf);


##LATITUDE BANDS 1600-1980
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/latbands.dat";
latbands<-scan(file=loc1,n=381*4,skip=1);
latbands<-t(array(latbands,dim=c(4,381)));
##col1 - year; col2 0-30 deg N; col3 - 30-70 deg N; col 4 - SH
locf<-"c:/climate/data/mann/latbands.tab";
save(latbands,file=locf);

##LATITUDE BANDS- RAW 1902-1993
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/latbands-raw.dat";
latbandsraw<-scan(file=loc1,n=3*92,skip=0);
latbandsraw<-t(array(latbandsraw,dim=c(3,92)));
##col1 - year; col2 0-30 deg N; col3 - 30-70 deg N
locf<-"c:/climate/data/mann/latbandsraw.tab";
save(latbandsraw,file=locf);

##REGIONAL 1650 to 1980
##col1-year; col2-Europe; col3- North America
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/regional.dat";
regional<-scan(file=loc1,n=3*381,skip=1);
regional<-t(array(regional,dim=c(3,381)));
locf<-"c:/climate/data/mann/regional.tab";
save(regional,file=locf);

##REGIONAL-RAW 1902 to 1993
##col1-year; col2-Europe; col3- North America
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/regional-raw.dat";
regionalraw<-scan(file=loc1,n=3*92,skip=1);
regionalraw<-t(array(regionalraw,dim=c(3,92)));
locf<-"c:/climate/data/mann/regionalraw.tab";
save(regionalraw,file=locf);

##SOLAR 1610 TO 1995
##col1-year; col2- solar (w m-2)
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/fig7-solar.dat";
solar<-scan(file=loc1,n=2*386,skip=1);
solar<-t(array(solar,dim=c(2,386)));
locf<-"c:/climate/data/mann/solar.tab";
save(solar,file=locf);

##VOLCANIC 1610 TO 1995
##col1-year; col2- volcanic (w m-2)
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/fig7-volcanic.dat";
volcanic<-scan(file=loc1,n=2*386,skip=1);
volcanic<-t(array(volcanic,dim=c(2,386)));
locf<-"c:/climate/data/mann/volcanic.tab";
save(volcanic,file=locf);

##CORRECTIONS 1780 TO 1895
##col1-year; col2-col7 - ?????
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/fig7-corrs.dat";
corrs<-scan(file=loc1,n=7*185,skip=1);
corrs<-t(array(corrs,dim=c(7,185)));
locf<-"c:/climate/data/mann/corrs.tab";
save(corrs,file=locf);

##CO2 1610 TO 1995
##col1-year; col2- co2 (w m-2)
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/fig7-co2.dat";
co2<-scan(file=loc1,n=2*386,skip=1);
co2<-t(array(co2,dim=c(2,386)));
locf<-"c:/climate/data/mann/co2.tab";
save(co2,file=locf);

##NH-OLD 1610 TO 1995
##col1-year; col2- NH -old
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/fig7-nh.dat";
nh.old<-scan(file=loc1,n=2*386,skip=3);
nh.old<-t(array(nh.old,dim=c(2,386)));
locf<-"c:/climate/data/mann/nh.old.tab";
save(nh.old,file=locf);

###NINO3 1650-1993
##col1-year; col2- NINO3 reconstruction; col3-NH -raw; col4- minus 1 sigma; col5- minus 2 sigma; col6- plus 1 sigma; col7 -plus 2 sigma
loc1<-"ftp://eclogite.geo.umass.edu/pub/mann/MANNETAL98/nino3.dat";
nino3<-scan(file=loc1,n=344*7,skip=1);
nino3<-t(array(nino3,dim=c(7,344)));
locf<-"c:/climate/data/mann/nino3.tab";
save(nino3,file=locf);


#MBH99 RECONSTRUCTIONS
base<-"ftp://ftp.ngdc.noaa.gov/paleo/contributions_by_author/mann1999/recons"
base.out<-"c:/climate/data/mann"

#NH.RAW
loc1<-file.path(base,"nhem-raw.dat")
mbh99.nh.raw<-read.table(loc1)
loc2<-file.path(base.out,"mbh99.nh.raw.tab")
save(mbh99.nh.raw,file=loc2)

#NH.RECON
loc1<-file.path(base,"nhem-recon.dat")
mbh99.nh.recon<-read.table(loc1)
loc2<-file.path(base.out,"mbh99.nh.recon.tab")
save(mbh99.nh.recon,file=loc2) #from 1000 to 1980

#STd.ERR
loc1<-file.path(base,"nhem-standerr-labeled.dat")
h<-readLines(loc1,n=-1)
h[1:15]

mbh99.nh.stderr.lab<-read.table(loc1,skip=1)
mbh99.nh.stderr.lab<-mbh99.nh.stderr.lab[,1:3]
loc2<-file.path(base.out,"mbh99.nh.stderr.lab.tab")
save(mbh99.nh.stderr.lab,file=loc2) #from 1000 to 1980

combine<-ts.union(a,b)
plot.ts(combine)