##CRUTEM2 COLLATION
	#this creatres a dataset of dimension 1853 x2592
	#5x5 gridcells: big hand 36 lat bands N to S, little hand 72 long bands from dateline W to dateline E
	#1815 months from goes from 1851 Jan to 2005 Mar (Aug 2005) 
	#the file has both temperature and count records: there are 108 temperature records and then 108 count records

	#this script takes Jones data from CRU, changes -9999 to NA and saves R-tables v - with dimension 1815 x 2592 (2592=72*36)
	#some of these steps take a few minutes since the dataset is large

#DO IN TWO PARTS
	N<-78
	N0<-N
	loc<-"c:/temp/crutem2/crutem2.dat" 
	v<-scan(loc,n=36*75*12*2*N)  #Read  5054400 items
	months<-12*N

	M<-length(v)
	v[1] #1851
	length(v)
	
	v<-array(v, dim=c(75,length(v)/75))     #t
	temp<-(v[3,]==1)
	v<-v[4:75,temp] #  72 33696
		##column 1 is year; column 2 is latitude group 1,,,36 north to south; column 3 is field in which 1=temperature; columns 4,..,75 are temperature anomalies for longitude groups 1,...72 from west to east (178.5W to 178.5E
		##NA values indicated by -9999

	#REPLACE -9999 WITH NA
	marker<- -9000
	temp<- (v<marker);
	v[temp]<-NA;
	rm(temp)

	v<-array(v,dim=c(36*72,months)) #v has dimension 2592xmonths
	v<-t(v)
	dim(v) # 936 2592
	save(v,file="c:/temp/crutem2.temp1.tab") #
		##this puts the time series for grid box (i,j) into column 36*(i-1)+(j-1)
	rm(v)

##SECOND HALF
	##SOMETIMES HELPS TO SHUT DOWN R, RESTART AND DO SECOND HALF
	N0<-78
	loc<-"c:/temp/crutem2/crutem2.dat" 
	N<-(155-N0)
	v<-scan(loc,n=36*75*12*2*N,skip=36*12*2*N0)  #4941000 items
	v<-v[(36*75*12*N0 +1):length(v)] #second half

	M<-length(v)
	v[1] #1929
	length(v)# 4941000
	
	v<-array(v, dim=c(75,length(v)/75))     #
	temp<-(v[3,]==1)
	v<-v[4:75,temp] #  72  32940
		##NA values indicated by -9999

	#REPLACE -9999 WITH NA
	marker<- -9000
	temp<- (v<marker);
	v[temp]<-NA;
	#this takes about a couple of seconds on my computer
	#save(v,file="c:/temp/crutem2.tab") #
	rm(temp)

	v<-array(v,dim=c(36*72,ncol(v)/36))) #
		#v has dimension 2592xmonths
	v<-t(v)
	dim(v) # 915 2592
	save(v,file="c:/temp/crutem2.temp.tab") #
		##this puts the time series for grid box (i,j) into column 36*(i-1)+(j-1)

###MERGE
	#sometimes helps to start and reload
	load("c:/temp/crutem2.temp.tab") 
	w<-v
	rm(v)

	load("c:/temp/crutem2.temp1.tab")
	v<-rbind(v,w)
	rm(w)
	dim(v) #1 1853 2592 #155 years long
	save(v,file="c:/temp/crutem2.tab") #