Difference between revisions of "DESY3 MICE2 source catalog"
(Created page with "notebook donde convierto los FITS files en parquet files: CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq ( `unique_gal_id` BIGINT, `z_cgal`...") |
|||
| Line 1: | Line 1: | ||
notebook donde convierto los FITS files en parquet files: | notebook donde convierto los FITS files en parquet files: | ||
| − | CREATE EXTERNAL TABLE jcarrete. | + | CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq ( |
`unique_gal_id` BIGINT, | `unique_gal_id` BIGINT, | ||
`z_cgal` DOUBLE, | `z_cgal` DOUBLE, | ||
| Line 28: | Line 28: | ||
| − | CREATE TABLE cosmohub. | + | CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c ( |
`unique_gal_id` BIGINT, | `unique_gal_id` BIGINT, | ||
`z_cgal` DOUBLE, | `z_cgal` DOUBLE, | ||
| Line 60: | Line 60: | ||
| − | INSERT OVERWRITE TABLE cosmohub. | + | INSERT OVERWRITE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c |
SELECT * | SELECT * | ||
| − | FROM jcarrete. | + | FROM jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq; |
| − | ANALYZE TABLE cosmohub. | + | ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS; |
| − | ANALYZE TABLE cosmohub. | + | ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS WITH COLUMNS; |
Revision as of 00:09, 22 April 2020
notebook donde convierto los FITS files en parquet files:
CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq (
`unique_gal_id` BIGINT, `z_cgal` DOUBLE, `ra_gal` DOUBLE, `dec_gal` DOUBLE, `g_sof` DOUBLE, `r_sof` DOUBLE, `i_sof` DOUBLE, `z_sof` DOUBLE, `z_dnf_mean_sof` FLOAT, `z_dnf_mc_sof` FLOAT, `des_asahi_full_g_true` FLOAT, `des_asahi_full_r_true` FLOAT, `des_asahi_full_i_true` FLOAT, `des_asahi_full_z_true` FLOAT, `kappa` FLOAT, `gamma1` FLOAT, `gamma2` FLOAT, `eps1` DOUBLE, `eps2` DOUBLE
) STORED AS PARQUET LOCATION '/user/jcarrete/data/mice/des_y3_mice2_source_catalog_IA_mocks'
CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c (
`unique_gal_id` BIGINT, `z_cgal` DOUBLE, `ra_gal` DOUBLE, `dec_gal` DOUBLE, `g_sof` DOUBLE, `r_sof` DOUBLE, `i_sof` DOUBLE, `z_sof` DOUBLE, `z_dnf_mean_sof` FLOAT, `z_dnf_mc_sof` FLOAT, `des_asahi_full_g_true` FLOAT, `des_asahi_full_r_true` FLOAT, `des_asahi_full_i_true` FLOAT, `des_asahi_full_z_true` FLOAT, `kappa` FLOAT, `gamma1` FLOAT, `gamma2` FLOAT, `eps1` DOUBLE, `eps2` DOUBLE
) CLUSTERED BY (
unique_gal_id)
INTO 4 BUCKETS ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
INSERT OVERWRITE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c
SELECT *
FROM jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq;
ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS;
ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS WITH COLUMNS;