#' ---
#' title: "Introduction to time series handling and  the Quantmod-Package"
#' author: "Dana Uhlig"
#' --



#getting a lot of Financial data via quantmod from YahooFinance and other sources

#install.packages("quantmod")
library(quantmod)   #loads also xts, zoo, ttr

#loading Dow Jones from yahoo 
getSymbols(Symbols = "^DJI") 
ls()

str(DJI)
head(DJI)
tail(DJI)
names(DJI)

DJI.long = getSymbols(Symbols = "^DJI", from="1995-01-01", auto.assign = FALSE)  
str(DJI.long)
head(DJI.long)
tail(DJI.long)
names(DJI.long)

#more symbols
getSymbols(c('AAPL','AMZN','VOW.DE','TSLA'),source="yahoo")

str(AAPL)
head(AAPL)
tail(AAPL)

#finding symbols via stockSymbols()
#all symbols from NYSE (typ. blue chip companies like AT&T, GM)
nyse.symbols = stockSymbols("NYSE")
dim(nyse.symbols)
head(nyse.symbols)
tail(nyse.symbols)

#saerching using regular expressions (grepl)
(gm.nyse = grep('General Motors',nyse.symbols[,2]))
nyse.symbols[gm.nyse,]
#-> symbol GM 

#or alternatively 
which(grepl('General Motors',nyse.symbols[,2]))


getSymbols('GM')

#all symbols from Nasdaq (typ. tech startups companies like Apple, Amazon, ..)
nasdaq.symbols = stockSymbols("NASDAQ")
dim(nasdaq.symbols)
head(nasdaq.symbols)
tail(nasdaq.symbols)

(amazon = grep('Amazon',nasdaq.symbols[,2]))
nasdaq.symbols[amazon,]

all.symbols = stockSymbols(exchange = c("AMEX","NASDAQ","NYSE"))

dim(all.symbols)
head(all.symbols)
tail(all.symbols)

str(all.symbols)

table(all.symbols$Sector)

#plot closin prices from Apple
plot(Cl(AAPL))

plot(AAPL$AAPL.Close)
#plot(AAPL)

chartSeries(VOW.DE)
chartSeries(TSLA,subset = 'last 3 years')
chartSeries(TSLA,subset = 'last 1 year')
chartSeries(TSLA,subset = 'last 250 days')

(vw.returns.2016 = dailyReturn(VOW.DE,subset="2016"))
(tsla.returns.2016 = dailyReturn(TSLA,subset="2016"))


#usual plots (generic) and calculus
plot(DJI)

x = DJI$DJI.Close

#x is xts object -> plot.xts based on chart_Series
plot(x)  
chart_Series(x)

#plain plot
plot(as.numeric(x),type="l")

plot(time(x),as.numeric(x),type="l")


#ret = log(x_t / x_{t-1}) = log(x_t) - log(x_{t-1})
x.ret = diff(log(x))
plot(x.ret)
hist(x.ret)

#same using quantmod function 
dji.ret = dailyReturn(DJI)
dji.ret.2009 = dailyReturn(DJI,subset="2009")

hist(dji.ret.2009)

dji.ret.2008.2009 = dailyReturn(DJI,subset="2008::2009")
plot(dji.ret.2008.2009)
hist(dji.ret.2008.2009)

dji.ret.y = periodReturn(DJI, period="yearly")

dji.ret.y

#same as annualReturn or yearlyReturn
dji.ret.a = annualReturn(DJI)
dji.ret.a - dji.ret.y

(test = allReturns(DJI))

plot(dji.ret)

plot(dji.ret, main="DJIA Returns", type="n")
lines(dji.ret)

###################################################################################################

#the package ggplot2 provides nicer graphics 
#install.packages("ggplot2")
library(ggplot2)
ggplot(data = dji.ret, aes(daily.returns)) + 
  geom_histogram(aes(y=..density..)) +
  geom_density(col="green") +
  stat_function(fun = dnorm, args=list(mean=mean(dji.ret),sd = sd(dji.ret)), colour = "red")

####################################################################################################

#financial charts using quantmod
chartSeries(DJI)
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red")
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2016::")
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2016::2017")
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2017")
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="last 12 months")
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="last 30 days")

chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2017",type = 'candlesticks')
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2017",type = 'matchsticks')
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2017",type = 'bars')
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2017",type = 'line')
chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red",subset="2017",type = 'auto')





#chart overlays and tecnical indicators:
#adding MACD (developed by Gerald Appel) popular price oscillator
chartSeries(Cl(DJI), theme = "white", up.col = "green", dn.col = "red",
            TA=" addMACD(fast = 8, slow = 11, signal = 6); addMACD(fast = 2, slow = 4, signal = 19)")

?chartSeries

chartSeries(DJI, theme = "white", up.col = "green", dn.col = "red", 
            TA="addRSI(n = 40); addRSI(n = 14)")


##

getSymbols("AAPL")

chartSeries(AAPL,subset = '2017')
plot(addMACD())
plot(addBBands())
plot(addMACD(fast = 8,slow = 11, signal = 6))


##
getSymbols(Symbols = "^GDAXI")  

head(GDAXI)

#introduction: 1988-07-01, Normierung 31. Dezember 1987  auf 1.000 
dax.long = getSymbols(Symbols = "^GDAXI",auto.assign = FALSE, from = "1985-01-01")
head(dax.long)


chartSeries(GDAXI, name = "Deutscher Aktien Index", theme = "white", up.col = "green", dn.col = "red", subset="last 1 year",
            TA="addSMA()") 

chartSeries(GDAXI, name = "Deutscher Aktien Index", theme = "white", up.col = "green", dn.col = "red", subset="last 12 months",
            TA="addSMA()") 


chartSeries(GDAXI,theme = "white", up.col = "green", dn.col = "red", subset="last 12 months",
            TA="addSMA()",type = "line" ) 

chartSeries(Cl(GDAXI), theme = "white", up.col = "green", dn.col = "red", subset="last 12 months",
            TA="addSMA()",type = "line" ) 


chartSeries(GDAXI,  theme = "white", up.col = "green", dn.col = "red", subset="last 12 months",
            TA="addSMA()",type="line")

chartSeries(GDAXI,  theme = "white", up.col = "green", dn.col = "red", subset="last 12 months",
            TA="addSMA()",type="line")
addSMA(on=1,n = 50, col = "blue")


chartSeries(GDAXI, name = "Deutscher Aktien Index", theme = "white", up.col = "green", dn.col = "red", subset="2007:2009",
            TA="addRSI(n = 40); addRSI(n = 14); 
            addMACD(fast = 8, slow = 11, signal = 6); 
            addMACD(fast = 2, slow = 4, signal = 19)")

chartSeries(GDAXI, name = "Deutscher Aktien Index", theme = "white", up.col = "green", dn.col = "red", subset="2007",
            TA="addRSI(n = 40); addRSI(n = 14); 
            addMACD(fast = 8, slow = 11, signal = 6); 
            addMACD(fast = 2, slow = 4, signal = 19)")


?EMA

####

#Bitcoin USD (BTC-USD)
getSymbols("BTC-USD")
ls()

`BTC-USD`

#problems with '-' sign in name -> ``

plot(`BTC-USD`)
#plot(BTC-USD)


#chartSeries(BTC-USD)
chartSeries(`BTC-USD`)

#own var
bitcoin_usd = getSymbols("BTC-USD",auto.assign = FALSE)
head(bitcoin_usd)

chartSeries(bitcoin_usd)
chartSeries(bitcoin_usd,subset = 'last 4 months')
chartSeries(bitcoin_usd,subset = '2018')
chartSeries(bitcoin_usd,subset = 'last 12 months')
chartSeries(bitcoin_usd, subset='2017-06::2018-05')


chartSeries(bitcoin_usd,theme=chartTheme('white'),subset = 'last 12 months')

chartSeries(bitcoin_usd,TA=NULL, subset='last 12 months',theme=chartTheme('white'))   #no volume
addVo() #add Volume
addBBands()
addMACD()
addCCI()
addChVol()
addRSI()


chartSeries(bitcoin_usd,name = "BTC-USD",TA=c(addVo(),addBBands()), subset='last 12 months')  #add volume and Bollinger Bands from TTR
addMACD()   #  add MACD indicator to current chart



#more Details see http://www.quantmod.com/examples/charting/

#######################################################################################################################
### other sources  (default = yahoo)
# 1. Googgle src doesn't work
# getSymbols("GOOG", src = "google")   

# 2. FRED: Federal Reserve Bank of St. Louis - Economic Research
#examples Gross Domestic Product (GDP) https://fred.stlouisfed.org/series/GDP/ 


#gdp.get = getSymbols("GDP",src = "FRED",auto.assign = FALSE)
#str(gdp.get)
#ls()

getSymbols("GDP",src = "FRED")

chartSeries(GDP)


#U.S. / Euro Foreign Exchange Rate (DEXUSEU)
getSymbols("DEXUSEU",src = "FRED")
chartSeries(DEXUSEU,theme="white")
chartSeries(1/DEXUSEU,theme="white")


# 10-Year Treasury Constant Maturity Rate (DGS10)
getSymbols("DGS10", src="FRED")
chartSeries(DGS10,theme="white")
chartSeries(DGS10,theme="white",subset="2007::")

#Civilian Unemployment Rate (UNRATE)
getSymbols("UNRATE", src="FRED")
chartSeries(UNRATE,theme="white")
chartSeries(UNRATE,theme="white",subset="2007::")

# 3-Month London Interbank Offered Rate (LIBOR), based on U.S. Dollar (USD3MTD156N)
getSymbols("USD3MTD156N", src="FRED")
chartSeries(USD3MTD156N,theme="white")
chartSeries(USD3MTD156N,theme="white",subset="2007::")


#3. OANDA 
getSymbols("USD/EUR",src="oanda",from="2005-01-01")


getSymbols("USD/EUR",src="oanda")
chartSeries(USDEUR,theme="white")
chartSeries(1/USDEUR,theme="white")
chartSeries(DEXUSEU,theme="white",subset = '2017-11-29::2018-05-27')


#see  https://www.quantmod.com/examples/intro/


#####  https://www.quandl.com/

#install.packages("Quandl")
library(Quandl)

gdp = Quandl(code = "FRED/GDP")
str(gdp)


# Import GDP data from FRED as xts
gdp_xts = Quandl(code = "FRED/GDP", type = "xts")
str(gdp_xts)
##


# Import GDP data from FRED as zoo
gdp_zoo = Quandl(code = "FRED/GDP", type = "zoo")

# Look at the structure of gdp_zoo
str(gdp_zoo)




#################################################################################

#multi assets - quantmod

tickers = c('GLD', 'GDX','USO')


getSymbols(tickers, src = 'yahoo', from = '2017-01-01')

chartSeries(GDX,theme="white",TA=NULL)
chartSeries(GLD,theme="white",TA=NULL)
chartSeries(USO,theme="white",TA=NULL)

chartSeries(rbind(GLD,GDX,USO), TA=NULL)

all = rbind(GLD,GDX,USO)

chartSeries(all,TA=NULL)

normalise_series <- function(xdat) xdat / coredata(xdat)[1]

# Define colour of default chart line to chart_Series in mytheme object
# which is passed to chart_Series:
mytheme = chart_theme()
mytheme$col$line.col = "darkgreen"
chart_Series(normalise_series(Cl(GLD)) , theme = mytheme)
add_TA(normalise_series(Cl(GDX)), on = 1, col = "red", lty = 3)
add_TA(normalise_series(Cl(USO)), on = 1, col = "blue", lty =2)

add_TA(RSI(Cl(GLD)), on = NA, col = "darkgreen")
add_TA(RSI(Cl(GDX)), on = 2, col = "red", lty = 3)
# Or add RSIs on different subpanels to improve readability of charts:
add_TA(RSI(Cl(USO)), on = 2, col = "blue", lty = 2)



############################################################################################################################

#creating own time series

z = ts(matrix(rt(200 * 8, df = 3), 200, 8),
       start = c(1961, 1), frequency = 12)
plot(z, yax.flip = TRUE)
plot(z, axes = FALSE, ann = FALSE, frame.plot = TRUE,
     mar.multi = c(0,0,0,0), oma.multi = c(1,1,5,1))
title("plot(ts(..), axes=FALSE, ann=FALSE, frame.plot=TRUE, mar..., oma...)")

str(z)
str(DJI)

#xts objects
zxts = as.xts(matrix(rt(200 * 8, df = 3), 200, 8),frequency = 12,order.by = seq(as.Date("2008-01-01"),length=200, by="weeks"))
head(zxts)

str(zxts)

data = rt(250, df=4)
dates = seq(as.Date("2017-01-01"), length=length(data), by = "days")

ts1 = xts(x=data, order.by=dates)

head(ts1)
tail(ts1)
str(ts1)

chartSeries(ts1)

deltat = 1/250
data2 = rnorm(250, mean = 0.03*deltat, sd = 0.2*deltat)
dates2 = seq(as.Date("2017-01-01"), length=length(data2), by = "days")

ts2 = xts(x=100*exp(data2), order.by=dates2)

chartSeries(ts2)


#################################
#install.packages("PerformanceAnalytics")
library(PerformanceAnalytics)
#Analyse und Darstellung der Portfolioperformance fuer Long
charts.PerformanceSummary(dji.ret)

chartSeries(dji.ret)
chartSeries(cumsum(dji.ret))

table.Drawdowns(dji.ret)
table.DownsideRisk(dji.ret)

sd(dji.ret)
quantile(dji.ret,0.95)

##############


##############################################################################################

####### many textbooks with own packages providing book examples and methods 

#Quelle: https://www.stat.pitt.edu/stoffer/tsa4/

#http://www.stat.pitt.edu/stoffer/tsa4/tsa4.htm

# https://www.springer.com/us/book/9783319524511

#-> ASTSA = Applied Statistical Time series Analysis



#install.packages("astsa")

library(astsa)

?astsa

#Johnson & Johnson Quarterly Earnings
plot(jj,type="o",main="Johnson & Johnson Quarterly Earnings")

plot(globtemp, type="o", ylab="Global Temperature Deviations")

plot(speech)

# El Niño and Fish Population (Figure 1.5)
#     monthly values of an environmental series called the Southern Oscillation Index (SOI)
#     and associated Recruitment (number of new fish) 
par(mfrow = c(2,1)) # set up the graphics
plot(soi, ylab="", xlab="", main="Southern Oscillation Index")
plot(rec, ylab="", xlab="", main="Recruitment")
par(mfrow = c(1,1)) # reset up the graphics

# fig 1.6
par(mfrow=c(2,1))
ts.plot(fmri1[,2:5], col=1:4, ylab="BOLD", main="Cortex")
ts.plot(fmri1[,6:9], col=1:4, ylab="BOLD", main="Thalamus & Cerebellum")
par(mfrow = c(1,1)) 

##############################################################################################