DESY3 MICE2 source catalog - Revision history

Jcarrete at 22:57, 15 May 2020

2020-05-15T22:57:15Z

Jcarrete at 00:13, 22 April 2020

2020-04-22T00:13:46Z

Jcarrete at 00:09, 22 April 2020

2020-04-22T00:09:11Z

Jcarrete: Created page with "notebook donde convierto los FITS files en parquet files: CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq ( `unique_gal_id` BIGINT, `z_cgal`..."

2020-04-22T00:05:22Z

Created page with "notebook donde convierto los FITS files en parquet files: CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq ( `unique_gal_id` BIGINT, `z_cgal`..."

New page

notebook donde convierto los FITS files en parquet files:

CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq (
`unique_gal_id` BIGINT,
`z_cgal` DOUBLE,
`ra_gal` DOUBLE,
`dec_gal` DOUBLE,
`g_sof` DOUBLE,
`r_sof` DOUBLE,
`i_sof` DOUBLE,
`z_sof` DOUBLE,
`z_dnf_mean_sof` FLOAT,
`z_dnf_mc_sof` FLOAT,
`des_asahi_full_g_true` FLOAT,
`des_asahi_full_r_true` FLOAT,
`des_asahi_full_i_true` FLOAT,
`des_asahi_full_z_true` FLOAT,
`kappa` FLOAT,
`gamma1` FLOAT,
`gamma2` FLOAT,
`eps1` DOUBLE,
`eps2` DOUBLE
)
STORED AS PARQUET
LOCATION '/user/jcarrete/data/mice/des_y3_mice2_source_catalog_IA_mocks'
;

CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c (
`unique_gal_id` BIGINT,
`z_cgal` DOUBLE,
`ra_gal` DOUBLE,
`dec_gal` DOUBLE,
`g_sof` DOUBLE,
`r_sof` DOUBLE,
`i_sof` DOUBLE,
`z_sof` DOUBLE,
`z_dnf_mean_sof` FLOAT,
`z_dnf_mc_sof` FLOAT,
`des_asahi_full_g_true` FLOAT,
`des_asahi_full_r_true` FLOAT,
`des_asahi_full_i_true` FLOAT,
`des_asahi_full_z_true` FLOAT,
`kappa` FLOAT,
`gamma1` FLOAT,
`gamma2` FLOAT,
`eps1` DOUBLE,
`eps2` DOUBLE
)
CLUSTERED BY (
unique_gal_id)
INTO 4 BUCKETS ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
;

INSERT OVERWRITE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c
SELECT *
FROM jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq

ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c COMPUTE STATISTICS;

ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c COMPUTE STATISTICS WITH COLUMNS;

@@ Line 51: / Line 51: @@
 CLUSTERED BY (
   unique_gal_id)
-INTO 4 BUCKETS ROW FORMAT SERDE
+INTO 8 BUCKETS ROW FORMAT SERDE
 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
 STORED AS INPUTFORMAT
@@ Line 66: / Line 66: @@
 ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS;
-ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS WITH COLUMNS;
+ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS FOR COLUMNS;

@@ Line 1: / Line 1: @@
-notebook donde convierto los FITS files en parquet files:
+== FIRST ITERATION ==
-CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq (
+There is a notebook donde convierto los FITS files en parquet files:
      `unique_gal_id` BIGINT,
      `z_cgal` DOUBLE,
@@ Line 21: / Line 25: @@
      `eps1` DOUBLE,
      `eps2` DOUBLE
-−
+    )
-STORED AS PARQUET
+    STORED AS PARQUET
-LOCATION '/user/jcarrete/data/mice/des_y3_mice2_source_catalog_IA_mocks'
+    LOCATION '/user/jcarrete/data/mice/des_y3_mice2_source_catalog_IA_mocks'
-−
+    ;
-CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c (
+    CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c (
     `unique_gal_id` BIGINT,
      `z_cgal` DOUBLE,
@@ Line 48: / Line 52: @@
      `eps1` DOUBLE,
      `eps2` DOUBLE
-−
+    )
-CLUSTERED BY (
+    CLUSTERED BY (
-  unique_gal_id)
+    unique_gal_id)
-INTO 8 BUCKETS ROW FORMAT SERDE
+    INTO 8 BUCKETS ROW FORMAT SERDE
-'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+    'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
-STORED AS INPUTFORMAT
+    STORED AS INPUTFORMAT
-'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+    'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
-OUTPUTFORMAT
+    OUTPUTFORMAT
-'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+    'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
-−
+    ;
-ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS;
+    ANALYZE TABLE cosmohub.scattz_sof_mice_feb3rd_maxmag22_5_part_c PARTITION(tomographic_bin) COMPUTE STATISTICS;
-ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS FOR COLUMNS;
+    ANALYZE TABLE cosmohub.scattz_sof_mice_feb3rd_maxmag22_5_part_c COMPUTE STATISTICS FOR COLUMNS;

@@ Line 1: / Line 1: @@
 notebook donde convierto los FITS files en parquet files:
-CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq (
+CREATE EXTERNAL TABLE jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq (
      `unique_gal_id` BIGINT,
      `z_cgal` DOUBLE,
@@ Line 28: / Line 28: @@
-CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c (
+CREATE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c (
     `unique_gal_id` BIGINT,
      `z_cgal` DOUBLE,
@@ Line 60: / Line 60: @@
-INSERT OVERWRITE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c
+INSERT OVERWRITE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c
 SELECT *
-FROM  jcarrete.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_pq
+FROM  jcarrete.scattZ_sof_mice_Feb3rd_maxmag22_5_pq;
-ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c COMPUTE STATISTICS;
+ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS;
-ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_zbin1_maxmag22_5_c COMPUTE STATISTICS WITH COLUMNS;
+ANALYZE TABLE cosmohub.scattZ_sof_mice_Feb3rd_maxmag22_5_c COMPUTE STATISTICS WITH COLUMNS;