Hive table with multiple SerDe
0
We have one HIVE table that is partitioned by date. It has currently Sequence file format, I want to convert it into Parquet Table.
Is it possible that we have new Partition with Parquet Serde, and older with Sequence format, so that I don't need to backfill it?
hive
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
add a comment |
0
We have one HIVE table that is partitioned by date. It has currently Sequence file format, I want to convert it into Parquet Table.
Is it possible that we have new Partition with Parquet Serde, and older with Sequence format, so that I don't need to backfill it?
hive
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
Why not make a separate table?CREATE TABLE t2 LIKE t STORED AS PARQUET?
– cricket_007
Jan 31 '18 at 22:43
@cricket_007 But then I need to backfill it, by converting Sequence files to Parquet files ( For 2-3 year of history data) . Also it will be different tablename that could break pipeline (that could be fixed by multiple ways)
– rajnish
Feb 1 '18 at 16:44
You cannot mix serdes. It's a table level setting, not partition level
– cricket_007
Feb 1 '18 at 18:39
add a comment |
0
0
0
We have one HIVE table that is partitioned by date. It has currently Sequence file format, I want to convert it into Parquet Table.
Is it possible that we have new Partition with Parquet Serde, and older with Sequence format, so that I don't need to backfill it?
hive
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
We have one HIVE table that is partitioned by date. It has currently Sequence file format, I want to convert it into Parquet Table.
Is it possible that we have new Partition with Parquet Serde, and older with Sequence format, so that I don't need to backfill it?
hive
hive
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
asked Jan 31 '18 at 18:07
rajnishrajnish
494615
494615
Why not make a separate table?CREATE TABLE t2 LIKE t STORED AS PARQUET?
– cricket_007
Jan 31 '18 at 22:43
@cricket_007 But then I need to backfill it, by converting Sequence files to Parquet files ( For 2-3 year of history data) . Also it will be different tablename that could break pipeline (that could be fixed by multiple ways)
– rajnish
Feb 1 '18 at 16:44
You cannot mix serdes. It's a table level setting, not partition level
– cricket_007
Feb 1 '18 at 18:39
add a comment |
Why not make a separate table?CREATE TABLE t2 LIKE t STORED AS PARQUET?
– cricket_007
Jan 31 '18 at 22:43
@cricket_007 But then I need to backfill it, by converting Sequence files to Parquet files ( For 2-3 year of history data) . Also it will be different tablename that could break pipeline (that could be fixed by multiple ways)
– rajnish
Feb 1 '18 at 16:44
You cannot mix serdes. It's a table level setting, not partition level
– cricket_007
Feb 1 '18 at 18:39
Why not make a separate table?
CREATE TABLE t2 LIKE t STORED AS PARQUET?– cricket_007
Jan 31 '18 at 22:43
Why not make a separate table?
CREATE TABLE t2 LIKE t STORED AS PARQUET?– cricket_007
Jan 31 '18 at 22:43
@cricket_007 But then I need to backfill it, by converting Sequence files to Parquet files ( For 2-3 year of history data) . Also it will be different tablename that could break pipeline (that could be fixed by multiple ways)
– rajnish
Feb 1 '18 at 16:44
@cricket_007 But then I need to backfill it, by converting Sequence files to Parquet files ( For 2-3 year of history data) . Also it will be different tablename that could break pipeline (that could be fixed by multiple ways)
– rajnish
Feb 1 '18 at 16:44
You cannot mix serdes. It's a table level setting, not partition level
– cricket_007
Feb 1 '18 at 18:39
You cannot mix serdes. It's a table level setting, not partition level
– cricket_007
Feb 1 '18 at 18:39
add a comment |
1 Answer
1
active
oldest
votes
0
create a external empty table with default serde(LazySimpleSerDe) and default stored(textfile).
add partition.
alter partition set fileformat(or set serde).
Hive LanguageManual DDL
CREATE EXTERNAL TABLE test(ip string, localTime string )
PARTITIONED BY (partition__hive__ STRING) location '/tmp/table/empty';
alter table test add partition (partition__hive__='p_0') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/08';
alter table test partition (partition__hive__='p_0') SET FILEFORMAT parquet;
alter table test add partition (partition__hive__='p_1') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/09';
alter table test partition (partition__hive__='p_1') SET SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f48548723%2fhive-table-with-multiple-serde%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
0
create a external empty table with default serde(LazySimpleSerDe) and default stored(textfile).
add partition.
alter partition set fileformat(or set serde).
Hive LanguageManual DDL
CREATE EXTERNAL TABLE test(ip string, localTime string )
PARTITIONED BY (partition__hive__ STRING) location '/tmp/table/empty';
alter table test add partition (partition__hive__='p_0') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/08';
alter table test partition (partition__hive__='p_0') SET FILEFORMAT parquet;
alter table test add partition (partition__hive__='p_1') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/09';
alter table test partition (partition__hive__='p_1') SET SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
add a comment |
0
create a external empty table with default serde(LazySimpleSerDe) and default stored(textfile).
add partition.
alter partition set fileformat(or set serde).
Hive LanguageManual DDL
CREATE EXTERNAL TABLE test(ip string, localTime string )
PARTITIONED BY (partition__hive__ STRING) location '/tmp/table/empty';
alter table test add partition (partition__hive__='p_0') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/08';
alter table test partition (partition__hive__='p_0') SET FILEFORMAT parquet;
alter table test add partition (partition__hive__='p_1') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/09';
alter table test partition (partition__hive__='p_1') SET SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
add a comment |
0
0
0
create a external empty table with default serde(LazySimpleSerDe) and default stored(textfile).
add partition.
alter partition set fileformat(or set serde).
Hive LanguageManual DDL
CREATE EXTERNAL TABLE test(ip string, localTime string )
PARTITIONED BY (partition__hive__ STRING) location '/tmp/table/empty';
alter table test add partition (partition__hive__='p_0') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/08';
alter table test partition (partition__hive__='p_0') SET FILEFORMAT parquet;
alter table test add partition (partition__hive__='p_1') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/09';
alter table test partition (partition__hive__='p_1') SET SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
create a external empty table with default serde(LazySimpleSerDe) and default stored(textfile).
add partition.
alter partition set fileformat(or set serde).
Hive LanguageManual DDL
CREATE EXTERNAL TABLE test(ip string, localTime string )
PARTITIONED BY (partition__hive__ STRING) location '/tmp/table/empty';
alter table test add partition (partition__hive__='p_0') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/08';
alter table test partition (partition__hive__='p_0') SET FILEFORMAT parquet;
alter table test add partition (partition__hive__='p_1') location 'hdfs://hdfsTest/hive/table/test/2018/11/21/09';
alter table test partition (partition__hive__='p_1') SET SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
edited Nov 22 '18 at 1:37
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
answered Nov 21 '18 at 9:03
Tianwang LiTianwang Li
11
11
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
add a comment |
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
Maybe you can explain with few sentences
– vahdet
Nov 21 '18 at 9:07
add a comment |
draft saved
draft discarded
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f48548723%2fhive-table-with-multiple-serde%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Why not make a separate table?
CREATE TABLE t2 LIKE t STORED AS PARQUET?– cricket_007
Jan 31 '18 at 22:43
@cricket_007 But then I need to backfill it, by converting Sequence files to Parquet files ( For 2-3 year of history data) . Also it will be different tablename that could break pipeline (that could be fixed by multiple ways)
– rajnish
Feb 1 '18 at 16:44
You cannot mix serdes. It's a table level setting, not partition level
– cricket_007
Feb 1 '18 at 18:39