Euclid mock production v1.7.17 deep correct v2
Jump to navigation
Jump to search
This is a new version for SC456 with a new set of shape parameters. The code has been modified taking suggestions from Lance Miller. The problems were reported in the Euclid redmine.
In order to correct them I use the notebook:
/nfs/pic.es/user/j/jcarrete/python_notebooks/Euclid_mock_pipeline_production-v.1.7.17_correct.ipynb
With the notebook I create several parquet files with the correct shape parameters. The input is the table jcarrete.flagship_SC456_1_7_17_deep_from_1_6_20.
Then I create an external parquet table in hive:
CREATE EXTERNAL TABLE `jcarrete.flagship_1_7_17_deep_correct_v2_pq`( `galaxy_id` bigint, `dominant_shape` bigint, `blanco_decam_i` double, `bulge_angle` double, `disk_angle` double, `median_major_axis` double, `scale_length` double, `bulge_fraction` double, `disk_length` double, `bulge_length` double, `ellipticity` double, `disk_axis_ratio` double, `bulge_axis_ratio` double, `halo_id` bigint) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' LOCATION 'hdfs://namenode01/user/jcarrete/data/euclid/flagship_1_7_17_deep_correct_v2_pq/mock' ;
Then this is the table to create the FITS files:
ADD JAR hdfs:///apps/cosmohub/lib/recarray-serde-LATEST-with-dependencies.jar;
CREATE TABLE `tallada.flagship_sc456_1_7_17_deep_correct_v2_from_1_6_20_jy_fits_fixed_r01`( SOURCE_ID bigint, RA float, DEC float, RA_MAG float, DEC_MAG float, Z_OBS float, TU_MAG_R01_SDSS_ABS float, TU_MAG_R01_SDSS float, SED_TEMPLATE float, EXT_LAW smallint, EBV float, HALPHA_LOGFLAM_EXT float, HBETA_LOGFLAM_EXT float, O2_LOGFLAM_EXT float, O3_LOGFLAM_EXT float, N2_LOGFLAM_EXT float, S2_LOGFLAM_EXT float, BULGE_FRACTION float, BULGE_LENGTH float, DISK_LENGTH float, DISK_AXIS_RATIO float, DISK_ANGLE float, KAPPA float, GAMMA1 float, GAMMA2 float, AV float, TU_FNU_VIS float, TU_FNU_Y_NISP float, TU_FNU_J_NISP float, TU_FNU_H_NISP float, TU_FNU_G_DECAM float, TU_FNU_R_DECAM float, TU_FNU_I_DECAM float, TU_FNU_Z_DECAM float, TU_FNU_U_MEGACAM float, TU_FNU_R_MEGACAM float, TU_FNU_G_JPCAM float, TU_FNU_I_PANSTARRS float, TU_FNU_Z_PANSTARRS float, TU_FNU_Z_HSC float, TU_FNU_G_GAIA float, TU_FNU_BP_GAIA float, TU_FNU_RP_GAIA float, TU_FNU_U_LSST float, TU_FNU_G_LSST float, TU_FNU_R_LSST float, TU_FNU_I_LSST float, TU_FNU_Z_LSST float, TU_FNU_Y_LSST float, TU_FNU_U_KIDS float, TU_FNU_G_KIDS float, TU_FNU_R_KIDS float, TU_FNU_I_KIDS float, TU_FNU_J_2MASS float, TU_FNU_H_2MASS float, TU_FNU_KS_2MASS float ) PARTITIONED BY ( hpix_5_nest bigint ) ROW FORMAT SERDE 'es.pic.astro.hadoop.serde.RecArraySerDe' STORED AS INPUTFORMAT 'es.pic.astro.hadoop.io.BinaryOutputFormat' OUTPUTFORMAT 'es.pic.astro.hadoop.io.BinaryOutputFormat' ;
This is the INSERT OVERWRITE command.
Note that for a partiotioned table, the last field needs to be the partitioned field, which in this case is the hpix_5_nest:
INSERT OVERWRITE TABLE tallada.flagship_sc456_1_7_17_deep_correct_v2_from_1_6_20_jy_fits_fixed_r01 PARTITION(hpix_5_nest)
SELECT
CAST( ((gal.halo_id * 10000) + gal.galaxy_id) AS bigint) AS SOURCE_ID,
CAST( gal.ra_gal AS float) AS RA,
CAST( gal.dec_gal AS float) AS DEC,
CAST( lens.ra_mag AS float) AS RA_MAG,
CAST( lens.dec_mag AS float) AS DEC_MAG,
CAST( gal.observed_redshift_gal AS float) AS Z_OBS,
CAST( gal.abs_mag_r01_evolved AS float) AS TU_MAG_R01_SDSS_ABS,
CAST( -2.5*log10(gal.sdss_r01) - 48.6 AS float) AS TU_MAG_R01_SDSS,
CAST( gal.sed_cosmos AS float) AS SED_TEMPLATE,
CAST( ROUND(gal.ext_curve_cosmos) AS smallint) AS EXT_LAW,
CAST( gal.ebv_cosmos AS float) AS EBV,
CAST( gal.logf_halpha_model3_ext AS float) AS HALPHA_LOGFLAM_EXT,
CAST( gal.logf_hbeta_model3_ext AS float) AS HBETA_LOGFLAM_EXT,
CAST( gal.logf_o2_model3_ext AS float) AS O2_LOGFLAM_EXT,
CAST( gal.logf_o3_model3_ext AS float) AS O3_LOGFLAM_EXT,
CAST( gal.logf_n2_model3_ext AS float) AS N2_LOGFLAM_EXT,
CAST( gal.logf_s2_model3_ext AS float) AS S2_LOGFLAM_EXT,
CAST( shape.bulge_fraction AS float) AS BULGE_FRACTION,
CAST( shape.bulge_length AS float) AS BULGE_LENGTH,
CAST( shape.disk_length AS float) AS DISK_LENGTH,
CAST( shape.disk_axis_ratio AS float) AS DISK_AXIS_RATIO,
CAST( shape.disk_angle AS float) AS DISK_ANGLE,
CAST( lens.kappa AS float) AS KAPPA,
CAST( lens.gamma1 AS float) AS GAMMA1,
CAST( lens.gamma2 AS float) AS GAMMA2,
CAST( gal.mw_extinction AS float) AS AV,
CAST( gal.euclid_vis_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_VIS,
CAST( gal.euclid_nisp_y_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_Y_NISP,
CAST( gal.euclid_nisp_j_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_J_NISP,
CAST( gal.euclid_nisp_h_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_H_NISP,
CAST( gal.blanco_decam_g_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_G_DECAM,
CAST( gal.blanco_decam_r_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_R_DECAM,
CAST( gal.blanco_decam_i_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_I_DECAM,
CAST( gal.blanco_decam_z_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_Z_DECAM,
CAST( gal.cfht_megacam_u_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_U_MEGACAM,
CAST( gal.cfht_megacam_r_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_R_MEGACAM,
CAST( gal.jst_jpcam_g_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_G_JPCAM,
CAST( gal.pan_starrs_i_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_I_PANSTARRS,
CAST( gal.pan_starrs_z_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_Z_PANSTARRS,
CAST( gal.subaru_hsc_z_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_Z_HSC,
CAST( gal.gaia_g_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_G_GAIA,
CAST( gal.gaia_bp_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_BP_GAIA,
CAST( gal.gaia_rp_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_RP_GAIA,
CAST( gal.lsst_u_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_U_LSST,
CAST( gal.lsst_g_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_G_LSST,
CAST( gal.lsst_r_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_R_LSST,
CAST( gal.lsst_i_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_I_LSST,
CAST( gal.lsst_z_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_Z_LSST,
CAST( gal.lsst_y_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_Y_LSST,
CAST( gal.kids_u_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_U_KIDS,
CAST( gal.kids_g_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_G_KIDS,
CAST( gal.kids_r_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_R_KIDS,
CAST( gal.kids_i_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_I_KIDS,
CAST( gal.2mass_j_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_J_2MASS,
CAST( gal.2mass_h_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_H_2MASS,
CAST( gal.2mass_ks_el_model3_odonnell_ext*1.e23 AS float) AS TU_FNU_KS_2MASS,
CAST( SHIFTRIGHT(gal.hpix_nest, (29-5)*2) AS bigint) AS hpix_5_nest
FROM jcarrete.flagship_SC456_1_7_17_deep_from_1_6_20 AS gal
JOIN jcarrete.flagship_1_7_17_deep_correct_v2_pq as shape
ON gal.halo_id = shape.halo_id
AND gal.galaxy_id = shape.galaxy_id
JOIN jcarrete.flagship_1_6_20_csv AS lens
ON ((gal.halo_id * 10000) + gal.galaxy_id) = lens.source_id
WHERE (gal.logf_halpha_model3_ext > -16 OR -2.5*log10(gal.euclid_nisp_h) - 48.6 < 26)
AND gal.disk_axis_ratio > 0.10865
;
Finally it is necessary to add the header for the FITS files and store them in the corresponding pnfs path.
The notebook is called: split in healpix pixels.ipynb, where it is necessary to config two paths:
path = '/apps/hive/warehouse/cosmohub.db/flagship_sc456_1_7_19_jy_fits/' pnfs_path = '/pnfs/pic.es/data/astro/euclid/disk/shared/SPV/GALAXIES/v33/