## ----setup, include = FALSE---------------------------------------------------
# only evaluate code if "NOT_CRAN"
NOT_CRAN <- identical(tolower(Sys.getenv("NOT_CRAN")), "true")

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)

if(NOT_CRAN){
  if(is.na(rtika::tika_jar())){ rtika::install_tika() }  
}


## ---- eval=NOT_CRAN-----------------------------------------------------------

library('rtika')
library('magrittr')

# Code to get ALL the files in my_path:

# my_path <- "~"
# batch <- file.path(my_path,
#                 list.files(path = my_path,
#                 recursive = TRUE))

# pipe the batch into tika_text() 
# to get plain text

# test files
batch <- c(
  system.file("extdata", "jsonlite.pdf", package = "rtika"),
  system.file("extdata", "curl.pdf", package = "rtika"),
  system.file("extdata", "table.docx", package = "rtika"),
  system.file("extdata", "xml2.pdf", package = "rtika"),
  system.file("extdata", "R-FAQ.html", package = "rtika"),
  system.file("extdata", "calculator.jpg", package = "rtika"),
  system.file("extdata", "tika.apache.org.zip", package = "rtika")
)

text <-  
    batch %>%
    tika_text() 

# normal syntax also works:
# text <- tika_text(batch)


## ---- eval=NOT_CRAN-----------------------------------------------------------
# Find which files had an issue
# Handle them if needed
batch[which(is.na(text))]

## ---- eval=NOT_CRAN-----------------------------------------------------------
length(text)

search <-
    text[grep(pattern = ' is ', x = text)]

length(search)

## ---- eval=NOT_CRAN-----------------------------------------------------------
download_directory <- tempfile('rtika_')

dir.create(download_directory)

urls <- c('https://tika.apache.org/',
          'https://cran.rstudio.com/web/packages/keras/keras.pdf')

downloaded <- 
    urls %>% 
    tika_fetch(download_directory)

# it will add the appropriate file extension to the downloads
downloaded


## ---- eval=NOT_CRAN-----------------------------------------------------------
# create a directory not already in use.
my_directory <-
   tempfile('rtika_')
                  
dir.create(my_directory)

# pipe the batch to tika_text()
batch %>%
tika_text(threads = 4,
          return = FALSE,
          output_dir = my_directory) 

# list all the file locations 
processed_files <- file.path(
                normalizePath(my_directory),
                list.files(path = my_directory,
                recursive = TRUE)
                )


## ---- eval=NOT_CRAN-----------------------------------------------------------
processed_files

## ---- eval=NOT_CRAN-----------------------------------------------------------
library('xml2')

# get XHTML text
html <- 
    batch %>%
    tika_html() %>%
    lapply(xml2::read_html)

# parse links from documents
links <-
    html %>%
    lapply(xml2::xml_find_all, '//a') %>%
    lapply(xml2::xml_attr, 'href')

sample(links[[1]],10)

## ---- eval=NOT_CRAN-----------------------------------------------------------
# Content-Type
html %>%
lapply(xml2::xml_find_first, '//meta[@name="Content-Type"]') %>%
lapply(xml2::xml_attr, 'content') %>%
unlist()

# Creation-Date
html %>%
lapply(xml2::xml_find_first, '//meta[@name="Creation-Date"]') %>%
lapply(xml2::xml_attr, 'content') %>%
unlist()


## ---- eval=NOT_CRAN-----------------------------------------------------------
library('jsonlite')
# batch <- system.file("extdata", "calculator.jpg", package = "rtika")

# a list of data.frames
metadata <-
    batch %>% 
    tika_json() %>%
    lapply(jsonlite::fromJSON)

# look at metadata for an image
str(metadata[[6]])


## ---- eval=NOT_CRAN-----------------------------------------------------------
metadata[[6]]$'geo:lat'
metadata[[6]]$'geo:long'

## ---- eval=NOT_CRAN-----------------------------------------------------------
# wget gets a webpage and other files. 
# sys::exec_wait('wget', c('--page-requisites', 'https://tika.apache.org/'))
# Put it all into a .zip file 
# sys::exec_wait('zip', c('-r', 'tika.apache.org.zip' ,'tika.apache.org'))
batch <- system.file("extdata", "tika.apache.org.zip", package = "rtika")

# a list of data.frames
metadata <-
    batch %>% 
    tika_json() %>%
    lapply(jsonlite::fromJSON)

# The structure is very long. See it on your own with: str(metadata)


## ---- eval=NOT_CRAN-----------------------------------------------------------
# the 'X-TIKA:embedded_resource_path' field
embedded_resource_path <- 
    metadata %>%
    lapply(function(x){ x$'X-TIKA:embedded_resource_path' }) 

embedded_resource_path

## ---- eval=NOT_CRAN-----------------------------------------------------------
content_type <-
    metadata %>%
    lapply(function(x){ x$'Content-Type' }) 

content_type

## ---- eval=NOT_CRAN-----------------------------------------------------------
content <- 
     metadata %>%
    lapply(function(x){ x$'X-TIKA:content' })

str(content)