匯入HDFS的資料到Hive

create external table if not exists finance.json_serde_optd_table (

retcode string,
retmsg string,
data array>)
row format serde 'org.apache.hive.hcatalog.data.jsonserde'
location 'hdfs:';
create table if not exists finance.tb_optd
asselect b.data.secid,
b.data.tradedate,
b.data.optid,
b.data.ticker,
b.data.secshortname,
b.data.exchangecd,
b.data.presettleprice,
b.data.precloseprice,
b.data.openprice,
b.data.highestprice,
b.data.lowestprice,
b.data.closeprice,
b.data.settlprice,
b.data.turnovervol,
b.data.turnovervalue,
b.data.openint
from finance.json_serde_optd_table lateral view explode(json_serde_optd_table.data) b as data;

%dep

z.load("/usr/hdp/2.4.2.0-258/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar");

// 定義匯入的hive物件集合

case class hiveconfig(database: string, modelname: string, hdfspath: string, schema: string, schema_tb: string);
var hiveconfiglist = list[hiveconfig]();

// 建立equd資料結構

// 定義json結構
val schema_json_equd_serde ="""  retcode string,
retmsg string,
data array>""";
var schema_equd ="""b.data.secid,
b.data.ticker,
b.data.secshortname,
b.data.exchangecd,
b.data.tradedate,
b.data.precloseprice,
b.data.actprecloseprice,
b.data.openprice,
b.data.highestprice,
b.data.lowestprice,
b.data.closeprice,
b.data.turnovervol,
b.data.turnovervalue,
b.data.dealamount,
b.data.turnoverrate,
b.data.accumadjfactor,
b.data.negmarketvalue,
b.data.marketvalue,
b.data.pe,
b.data.pe1,
b.data.pb,
b.data.isopen""";
hiveconfiglist  = hiveconfiglist :+ hiveconfig("finance", "equd", "hdfs:", schema_json_equd_serde, schema_equd);

// 建立idxd資料結構

// 定義json結構
val schema_json_idxd_serde ="""  retcode string,
retmsg string,
data array>""";
var schema_idxd ="""b.data.indexid,
b.data.tradedate,
b.data.ticker,
b.data.porgfullname,
b.data.secshortname,
b.data.exchangecd,
b.data.precloseindex,
b.data.openindex,
b.data.lowestindex,
b.data.highestindex,
b.data.closeindex,
b.data.turnovervol,
b.data.turnovervalue,
b.data.chg,
b.data.chgpct""";
hiveconfiglist = hiveconfiglist :+ hiveconfig("finance", "idxd", "hdfs:", schema_json_idxd_serde, schema_idxd);

// 迴圈載入資料中

def loaddatatohive(args:hiveconfig)
}  hiveconfiglist.size;
hiveconfiglist.foreach ;

用splitjson 來提取、分隔 data 陣列

nifi 中國社群 qq群：595034369

hive 本地 hdfs資料匯入

1.1匯入內部表 1 本地或者hdfs匯入 load data local inpath filepath overwrite into tabletablename partition partcol1 val1,partcol2 val2 區別是看有無關鍵字local，有local表示從本地路徑...

如何快速把hdfs資料動態匯入到hive表

create external table if not exists sensitop.equd json tmp retcode string,retmsg string,data array row format serde org.apache.hive.hcatalog.data.json...

hive匯入資料到hbase

hive有一張表user tag detail，表中資料約1.3億，需要將改表資料匯入到hbase 嘗試了兩種方式 建立關聯表 create table hbase user tag detail id string,name string 插入資料 insert overwrite table h...