Merge pull request #371 from daniels290813/hive_url_fix

[data-ingestion-and-preparation] fixing hive url
v3io · Sep 12, 2022 · 897e82e · 897e82e
2 parents 06baae3 + 389fa1e
commit 897e82e
Show file tree

Hide file tree

Showing 2 changed files with 181 additions and 54 deletions.
diff --git a/data-ingestion-and-preparation/csv-to-hive.ipynb b/data-ingestion-and-preparation/csv-to-hive.ipynb
@@ -18,13 +18,13 @@
      "text": [
       "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
       "                                 Dload  Upload   Total   Spent    Left  Speed\n",
-      "100  861k  100  861k    0     0  7425k      0 --:--:-- --:--:-- --:--:-- 7425k\n",
+      "100  861k  100  861k    0     0   380k      0  0:00:02  0:00:02 --:--:--  380k\n",
       "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
       "                                 Dload  Upload   Total   Spent    Left  Speed\n",
-      "100  861k  100  861k    0     0  7362k      0 --:--:-- --:--:-- --:--:-- 7362k\n",
+      "100  861k  100  861k    0     0   356k      0  0:00:02  0:00:02 --:--:--  356k\n",
       "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
       "                                 Dload  Upload   Total   Spent    Left  Speed\n",
-      "100  861k  100  861k    0     0  7760k      0 --:--:-- --:--:-- --:--:-- 7760k\n"
+      "100  861k  100  861k    0     0   306k      0  0:00:02  0:00:02 --:--:--  306k\n"
      ]
     }
    ],
@@ -71,7 +71,7 @@
    "outputs": [],
    "source": [
     "from pyspark.sql import SparkSession\n",
-    "spark = SparkSession.builder.appName(\"Import parquet schema to hive\").config(\"hive.metastore.uris\", \"thrift://hive:9083\").enableHiveSupport().getOrCreate()"
+    "spark = SparkSession.builder.appName(\"Import parquet schema to hive\").config(\"hive.metastore.uris\", \"thrift://trino-hive:9083\").enableHiveSupport().getOrCreate()"
    ]
   },
   {
@@ -131,7 +131,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CREATE EXTERNAL TABLE IF NOT EXISTS default.csv_table(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID int, Date timestamp, Time string, StartPrice double, MaxPrice double, MinPrice double, EndPrice double, TradedVolume int, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/admin/examples/demo.csv' TBLPROPERTIES('skip.header.line.count'='1') \n"
+      "CREATE EXTERNAL TABLE IF NOT EXISTS default.csv_table(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID int, Date string, Time string, StartPrice double, MaxPrice double, MinPrice double, EndPrice double, TradedVolume int, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/dani/examples/demo.csv' TBLPROPERTIES('skip.header.line.count'='1') \n"
      ]
     }
    ],
@@ -157,7 +157,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CREATE EXTERNAL TABLE IF NOT EXISTS default.table_from_dir2(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID string, Date string, Time string, StartPrice string, MaxPrice string, MinPrice string, EndPrice string, TradedVolume string, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/admin/examples/csvs/' TBLPROPERTIES('skip.header.line.count'='1') \n"
+      "CREATE EXTERNAL TABLE IF NOT EXISTS default.table_from_dir2(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID string, Date string, Time string, StartPrice string, MaxPrice string, MinPrice string, EndPrice string, TradedVolume string, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/dani/examples/csvs/' TBLPROPERTIES('skip.header.line.count'='1') \n"
      ]
     }
    ],
@@ -189,10 +189,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CREATE EXTERNAL TABLE IF NOT EXISTS default.demo1(id int, street string, city string, zip int, state string, beds int, baths int, sq__ft int, type string, sale_date string, price int, latitude double, longitude double) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/admin/examples/csvs/demo1.csv' TBLPROPERTIES('skip.header.line.count'='1') \n",
-      "CREATE EXTERNAL TABLE IF NOT EXISTS default.demo2(id int, street string, city string, zip int, state string, beds int, baths int, sq__ft int, type string, sale_date string, price int, latitude double, longitude double) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/admin/examples/csvs/demo2.csv' TBLPROPERTIES('skip.header.line.count'='1') \n",
-      "CREATE EXTERNAL TABLE IF NOT EXISTS default.example1(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID int, Date timestamp, Time string, StartPrice double, MaxPrice double, MinPrice double, EndPrice double, TradedVolume int, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/admin/examples/csvs/example1.csv' TBLPROPERTIES('skip.header.line.count'='1') \n",
-      "CREATE EXTERNAL TABLE IF NOT EXISTS default.example2(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID int, Date timestamp, Time string, StartPrice double, MaxPrice double, MinPrice double, EndPrice double, TradedVolume int, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/admin/examples/csvs/example2.csv' TBLPROPERTIES('skip.header.line.count'='1') \n"
+      "CREATE EXTERNAL TABLE IF NOT EXISTS default.demo1(id int, street string, city string, zip int, state string, beds int, baths int, sq__ft int, type string, sale_date string, price int, latitude double, longitude double) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/dani/examples/csvs/demo1.csv' TBLPROPERTIES('skip.header.line.count'='1') \n",
+      "CREATE EXTERNAL TABLE IF NOT EXISTS default.demo2(id int, street string, city string, zip int, state string, beds int, baths int, sq__ft int, type string, sale_date string, price int, latitude double, longitude double) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/dani/examples/csvs/demo2.csv' TBLPROPERTIES('skip.header.line.count'='1') \n",
+      "CREATE EXTERNAL TABLE IF NOT EXISTS default.example1(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID int, Date string, Time string, StartPrice double, MaxPrice double, MinPrice double, EndPrice double, TradedVolume int, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/dani/examples/csvs/example1.csv' TBLPROPERTIES('skip.header.line.count'='1') \n",
+      "CREATE EXTERNAL TABLE IF NOT EXISTS default.example2(ISIN string, Mnemonic string, SecurityDesc string, SecurityType string, Currency string, SecurityID int, Date string, Time string, StartPrice double, MaxPrice double, MinPrice double, EndPrice double, TradedVolume int, NumberOfTrades int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION 'v3io://users/dani/examples/csvs/example2.csv' TBLPROPERTIES('skip.header.line.count'='1') \n"
      ]
     }
    ],
@@ -224,22 +224,22 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+------------+\n",
-      "|databaseName|\n",
-      "+------------+\n",
-      "|     default|\n",
-      "+------------+\n",
+      "+---------+\n",
+      "|namespace|\n",
+      "+---------+\n",
+      "|  default|\n",
+      "+---------+\n",
       "\n",
-      "+--------+---------------+-----------+\n",
-      "|database|      tableName|isTemporary|\n",
-      "+--------+---------------+-----------+\n",
-      "| default|      csv_table|      false|\n",
-      "| default|          demo1|      false|\n",
-      "| default|          demo2|      false|\n",
-      "| default|       example1|      false|\n",
-      "| default|       example2|      false|\n",
-      "| default|table_from_dir2|      false|\n",
-      "+--------+---------------+-----------+\n",
+      "+---------+---------------+-----------+\n",
+      "|namespace|      tableName|isTemporary|\n",
+      "+---------+---------------+-----------+\n",
+      "|  default|      csv_table|      false|\n",
+      "|  default|          demo1|      false|\n",
+      "|  default|          demo2|      false|\n",
+      "|  default|       example1|      false|\n",
+      "|  default|       example2|      false|\n",
+      "|  default|table_from_dir2|      false|\n",
+      "+---------+---------------+-----------+\n",
       "\n"
      ]
     }
@@ -259,7 +259,7 @@
     {
      "data": {
       "text/plain": [
-       "DataFrame[ISIN: string, Mnemonic: string, SecurityDesc: string, SecurityType: string, Currency: string, SecurityID: int, Date: timestamp, Time: string, StartPrice: double, MaxPrice: double, MinPrice: double, EndPrice: double, TradedVolume: int, NumberOfTrades: int]"
+       "DataFrame[ISIN: string, Mnemonic: string, SecurityDesc: string, SecurityType: string, Currency: string, SecurityID: int, Date: string, Time: string, StartPrice: double, MaxPrice: double, MinPrice: double, EndPrice: double, TradedVolume: int, NumberOfTrades: int]"
       ]
      },
      "execution_count": 10,
@@ -310,7 +310,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.7.6"
   }
  },
  "nbformat": 4,