ulimit -u unlimited

ulimit -n 1039360

cd cxoracle

spark-shell --driver-memory 14G --executor-memory 14G --jars ojdbc6.jar --conf

spark.sql.shuffle.partitions=800 --conf spark.default.parallelism=800 --conf
spark.local.dir=/clarolab/tempspark2

-----tunnel: port 8888 destination limdatalabf02.claro.pe:4040 ruta:

http://localhost:8888/jobs/

import org.apache.spark.sql.SparkSession
import spark.implicits._
import org.apache.spark.sql.DataFrame
import org.apache.spark.SparkContext;
import org.apache.spark.SparkConf ;
import org.apache.spark.sql.SQLContext

val spark = SparkSession.builder().appName("Spark SQL basic

example").config("spark.some.config.option", "some-value").getOrCreate()

val conf = new SparkConf()

val sc = new SparkContext(conf)
val sqlContext = new SQLContext(sc)

---formato a las columnas (Transformacion de datos) NECESARIO EN LA TRANSFORMACION

DE DATOS
--import org.apache.spark.sql.functions._
--import org.apache.spark.sql.types._

----LEER DATOS DE UNA TABLA EN ORACLE

val Query = "(SELECT TIM_NUMBER,NUMBER_B,RECORD_TYPE,CHARGING_START_TIME,ANIO,MES
from C18925.C19936_LAV_CDRDWH_20200905 where rownum<=200) emp"
val df = sqlContext.read.format("jdbc").options(Map("url" ->
"jdbc:oracle:thin:CHURNLAB/churndata$1@//scan-dwh.tim.com.pe:1521/DWM", "user" ->
"CHURNLAB", "churndata$1" -> "impetus", "dbtable" -> Query, "driver" ->
"oracle.jdbc.driver.OracleDriver")).load()
--val df =
spark.read.format("jdbc").option("url","jdbc:oracle:thin:CHURNLAB/churndata$1@//sca
n-
dwh.tim.com.pe:1521/DWM").option("dbtable",Query).option("user","CHURNLAB").option(
"password","churndata$1").option("driver","oracle.jdbc.driver.OracleDriver").load()

---INSERTAR EN ARCHIVO PARQUET

df.write.parquet("/clarolab/tempspark2/repositorio/prueba2.parquet")

---LEER DATOS DE UN ARCHIVO PARQUET EN UNA VARIABLE

var DATOS_DWH =
spark.read.parquet("/clarolab/tempspark2/repositorio/prueba2.parquet")

--DATOS_DWH.printSchema (CONSULTAR ESTRUCTURA)

--DATOS_DWH.show (MUESTRA LOS REGISTROS)
--DATOS.DWH.count (MUESTRA LA CANTIDAD DE REGISTROS)

---SELECCIONA CAMPOS QUE SE DESEA USAR

val DATOS_DWH_C=DATOS_DWH.select("TIM_NUMBER" , "NUMBER_B" , "RECORD_TYPE" ,
"CHARGING_START_TIME" , "ANIO" , "MES")

----CONVERSION DE TIPO DE DATOS

val DATOS_DWH_FORMAT=DATOS_DWH_C.withColumn("ANIO",
col("ANIO").cast(IntegerType)).withColumn("MES", col("MES").cast(IntegerType))

----GUARDA EL RESULTADO EN MEMORIA PARA SER USADO POR UN SPARK SQL

DATOS_DWH_FORMAT.createOrReplaceTempView("DATOS_DWH_FORMAT")

----HACER UNA CONSULTA SPARK SQL

val DF_DATOS_DWH_FORMAT = spark.sql("SELECT
TO_DATE(CAST(UNIX_TIMESTAMP(substr(CHARGING_START_TIME,7,2)||'/'||
substr(CHARGING_START_TIME,5,2)||'/'||substr(CHARGING_START_TIME,1,4),
'dd/MM/yyyy') AS TIMESTAMP)) AS FECHA,* FROM DATOS_DWH_FORMAT")

---GUARDA EL RESULTADO EN MEMORIA PARA SER USADO POR UN SPARK SQL

DF_DATOS_DWH_FORMAT.createOrReplaceTempView("DF_DATOS_DWH_FORMAT")

----LIBERAR MEMORIA CACHE

Sudo sh –c ‘echo 1 >/proc/sys/vm/drop_cache’
Sudo sh –c ‘echo 2 >/proc/sys/vm/drop_cache’
Sudo sh –c ‘echo 3 >/proc/sys/vm/drop_cache’

---Este código es más actualizado

sync; echo 1 >/proc/sys/vm/drop_caches
sync; echo 2 >/proc/sys/vm/drop_caches
sync; echo 3 >/proc/sys/vm/drop_caches

sync && echo 3 >/proc/sys/vm/drop_caches

---INICIAR Y DETENER SERVICIO DE JUPYTER

sudo systemctl stop jupyterhub
sudo systemctl start jupyterhub
sudo systemctl status jupyterhub

-----CADENAS DE CONEXION A BD ORACLE

---DWO
DWO.TIM.COM.PE =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = scan-dwo.tim.com.pe)(PORT = 1521))
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = DWO)
)
)

--REPTDM
REPTDM.TIM.COM.PE =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = scan-dwh.tim.com.pe)(PORT = 1521))
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = REPTDM)
)
)
----TABLA DE VALERIA
val Query2 = "(select msisdn, num_tra, monto_c_igv from
c17383.paquetes_datos_prepago) emp2"
val df2 = sqlContext.read.format("jdbc").options(Map("url" ->
"jdbc:oracle:thin:CHURNLAB/churndata$1@//scan-dwo.tim.com.pe:1521/DWO", "user" ->
"CHURNLAB", "churndata$1" -> "impetus", "dbtable" -> Query2, "driver" ->
"oracle.jdbc.driver.OracleDriver")).load()