如何快速把hdfs資料動態匯入到hive表

,,,,,,,,,,,]}

create external table if not exists sensitop.equd_json_tmp (

retcode string,
retmsg string,
data array>)
row format serde 'org.apache.hive.hcatalog.data.jsonserde'
location 'hdfs:';

需要把上面表裡陣列裡的資料一條一條放入這個表：

create table if not exists sensitop.equd_h(

secid string,
ticker string,
secshortname string,
exchangecd string,
tradedate date,
precloseprice double,
actprecloseprice double,
openprice double,
highestprice double,
lowestprice double,
closeprice double,
turnovervol double,
turnovervalue double,
dealamount int,
turnoverrate double,
accumadjfactor double,
negmarketvalue double,
marketvalue double,
pe double,
pe1 double,
pb double,
isopen int)
partitioned by (year string)
row format serde 'org.apache.hive.hcatalog.data.jsonserde'

然後新建乙個最張表

create table if not exists sensitop.equd(

secid string,
ticker string,
secshortname string,
exchangecd string,
tradedate date,
precloseprice double,
actprecloseprice double,
openprice double,
highestprice double,
lowestprice double,
closeprice double,
turnovervol double,
turnovervalue double,
dealamount int,
turnoverrate double,
accumadjfactor double,
negmarketvalue double,
marketvalue double,
pe double,
pe1 double,
pb double,
isopen int)
partitioned by (year string)

注意：這裡的字段順序和上面臨時表的順序要一致。

insert overwrite table sensitop.equd_tmp

partition (year='2016')
select b.dt.secid,
b.dt.ticker,
b.dt.secshortname,
b.dt.exchangecd,
b.dt.tradedate,
b.dt.precloseprice,
b.dt.actprecloseprice,
b.dt.openprice,
b.dt.highestprice,
b.dt.lowestprice,
b.dt.closeprice,
b.dt.turnovervol,
b.dt.turnovervalue,
b.dt.dealamount,
b.dt.turnoverrate,
b.dt.accumadjfactor,
b.dt.negmarketvalue,
b.dt.marketvalue,
b.dt.pe,
b.dt.pe1,
b.dt.pb,
b.dt.isopen
from sensitop.equd_json_tmp lateral view explode(equd_json_tmp.data) b as dt
where dt.tradedate >= '2016-01-01' and dt.tradedate <= '2016-12-31';

insert overwrite table sensitop.equd

partition (year='2016')
select secid,
ticker,
secshortname,
exchangecd,
tradedate,
precloseprice,
actprecloseprice,
openprice,
highestprice,
lowestprice,
closeprice,
turnovervol,
turnovervalue,
dealamount,
turnoverrate,
accumadjfactor,
negmarketvalue,
marketvalue,
pe,pe1,
pb,isopen
from sensitop.equd_tmp dt
where year = '2016';

這裡有二個分支，左邊乙個是每天20：00更新當年的partion； 右邊乙個是更新1990 到 2015 年的資料，而且只需要更新一次。

insert overwrite table sensitop.equd_h

partition (year='$')
select b.dt.secid,
b.dt.ticker,
b.dt.secshortname,
b.dt.exchangecd,
b.dt.tradedate,
b.dt.precloseprice,
b.dt.actprecloseprice,
b.dt.openprice,
b.dt.highestprice,
b.dt.lowestprice,
b.dt.closeprice,
b.dt.turnovervol,
b.dt.turnovervalue,
b.dt.dealamount,
b.dt.turnoverrate,
b.dt.accumadjfactor,
b.dt.negmarketvalue,
b.dt.marketvalue,
b.dt.pe,
b.dt.pe1,
b.dt.pb,
b.dt.isopen
from sensitop.equd_json_tmp lateral view explode(equd_json_tmp.data) b as dt
where dt.tradedate >= '$-01-01' and dt.tradedate <= '$-12-31'

insert overwrite table sensitop.equd

partition (year='$')
select secid,
ticker,
secshortname,
exchangecd,
tradedate,
precloseprice,
actprecloseprice,
openprice,
highestprice,
lowestprice,
closeprice,
turnovervol,
turnovervalue,
dealamount,
turnoverrate,
accumadjfactor,
negmarketvalue,
marketvalue,
pe,pe1,
pb,isopen
from sensitop.equd_tmp dt
where year = '$'

hive ,從hdfs把資料檔案load匯入到表

hive load data inpath hdfs ns1 abc sales info hello sales info.txt overwrite into table sales info partition dt 2019 04 26 1 原資料檔案 已經不存在了，是從原路徑移動到了新路徑...

hive 本地 hdfs資料匯入

1.1匯入內部表 1 本地或者hdfs匯入 load data local inpath filepath overwrite into tabletablename partition partcol1 val1,partcol2 val2 區別是看有無關鍵字local，有local表示從本地路徑...

如何把excel資料匯入到mysql資料庫中

最近做的工作涉及到把excel資料匯入到mysql資料庫中,看來一些mysqlimport,phpmyadmin命令,但是不怎麼會用.就決定自己寫指令碼解決.先把excel資料檔案儲存成csv格式的檔案,然後寫了下面的簡單perl指令碼倒入mysql資料庫中.需要注意用你的mysql資料庫表的結構替...