Jcarrete: Created page with "I take the data from Giorgio's cephs directory I use some notebook to convert the hdf5 file to parquet file and store it in the /hdfs I use the script to create external tab..."

2020-04-08T10:26:16Z

Created page with "I take the data from Giorgio's cephs directory I use some notebook to convert the hdf5 file to parquet file and store it in the /hdfs I use the script to create external tab..."

New page

I take the data from Giorgio's cephs directory

I use some notebook to convert the hdf5 file to parquet file and store it in the /hdfs

I use the script to create external tables in the software astro area

CREATE EXTERNAL TABLE jcarrete.lc5_field1_63_pq (
`BCDM` FLOAT,
`GalaxyID` BIGINT,
`app_PAU-4550_o_tot_e` FLOAT,
`app_PAU-4650_o_tot_e` FLOAT,
`app_PAU-4750_o_tot_e` FLOAT,
`app_PAU-4850_o_tot_e` FLOAT,
`app_PAU-4950_o_tot_e` FLOAT,
`app_PAU-5050_o_tot_e` FLOAT,
`app_PAU-5150_o_tot_e` FLOAT,
`app_PAU-5250_o_tot_e` FLOAT,
`app_PAU-5350_o_tot_e` FLOAT,
`app_PAU-5450_o_tot_e` FLOAT,
`app_PAU-5550_o_tot_e` FLOAT,
`app_PAU-5650_o_tot_e` FLOAT,
`app_PAU-5750_o_tot_e` FLOAT,
`app_PAU-5850_o_tot_e` FLOAT,
`app_PAU-5950_o_tot_e` FLOAT,
`app_PAU-6050_o_tot_e` FLOAT,
`app_PAU-6150_o_tot_e` FLOAT,
`app_PAU-6250_o_tot_e` FLOAT,
`app_PAU-6350_o_tot_e` FLOAT,
`app_PAU-6450_o_tot_e` FLOAT,
`app_PAU-6550_o_tot_e` FLOAT,
`app_PAU-6650_o_tot_e` FLOAT,
`app_PAU-6750_o_tot_e` FLOAT,
`app_PAU-6850_o_tot_e` FLOAT,
`app_PAU-6950_o_tot_e` FLOAT,
`app_PAU-7050_o_tot_e` FLOAT,
`app_PAU-7150_o_tot_e` FLOAT,
`app_PAU-7250_o_tot_e` FLOAT,
`app_PAU-7350_o_tot_e` FLOAT,
`app_PAU-7450_o_tot_e` FLOAT,
`app_PAU-7550_o_tot_e` FLOAT,
`app_PAU-7650_o_tot_e` FLOAT,
`app_PAU-7750_o_tot_e` FLOAT,
`app_PAU-7850_o_tot_e` FLOAT,
`app_PAU-7950_o_tot_e` FLOAT,
`app_PAU-8050_o_tot_e` FLOAT,
`app_PAU-8150_o_tot_e` FLOAT,
`app_PAU-8250_o_tot_e` FLOAT,
`app_PAU-8350_o_tot_e` FLOAT,
`app_PAU-8450_o_tot_e` FLOAT,
`app_SDSS-g_o_tot_ext` FLOAT,
`app_SDSS-i_o_tot_ext` FLOAT,
`app_SDSS-r_o_tot_ext` FLOAT,
`app_SDSS-u_o_tot_ext` FLOAT,
`app_SDSS-z_o_tot_ext` FLOAT,
`dec` DOUBLE,
`idrep` INT,
`mag_PAU-4550_r_tot_ext` FLOAT,
`mag_PAU-4650_r_tot_ext` FLOAT,
`mag_PAU-4750_r_tot_ext` FLOAT,
`mag_PAU-4850_r_tot_ext` FLOAT,
`mag_PAU-4950_r_tot_ext` FLOAT,
`mag_PAU-5050_r_tot_ext` FLOAT,
`mag_PAU-5150_r_tot_ext` FLOAT,
`mag_PAU-5250_r_tot_ext` FLOAT,
`mag_PAU-5350_r_tot_ext` FLOAT,
`mag_PAU-5450_r_tot_ext` FLOAT,
`mag_PAU-5550_r_tot_ext` FLOAT,
`mag_PAU-5650_r_tot_ext` FLOAT,
`mag_PAU-5750_r_tot_ext` FLOAT,
`mag_PAU-5850_r_tot_ext` FLOAT,
`mag_PAU-5950_r_tot_ext` FLOAT,
`mag_PAU-6050_r_tot_ext` FLOAT,
`mag_PAU-6150_r_tot_ext` FLOAT,
`mag_PAU-6250_r_tot_ext` FLOAT,
`mag_PAU-6350_r_tot_ext` FLOAT,
`mag_PAU-6450_r_tot_ext` FLOAT,
`mag_PAU-6550_r_tot_ext` FLOAT,
`mag_PAU-6650_r_tot_ext` FLOAT,
`mag_PAU-6750_r_tot_ext` FLOAT,
`mag_PAU-6850_r_tot_ext` FLOAT,
`mag_PAU-6950_r_tot_ext` FLOAT,
`mag_PAU-7050_r_tot_ext` FLOAT,
`mag_PAU-7150_r_tot_ext` FLOAT,
`mag_PAU-7250_r_tot_ext` FLOAT,
`mag_PAU-7350_r_tot_ext` FLOAT,
`mag_PAU-7450_r_tot_ext` FLOAT,
`mag_PAU-7550_r_tot_ext` FLOAT,
`mag_PAU-7650_r_tot_ext` FLOAT,
`mag_PAU-7750_r_tot_ext` FLOAT,
`mag_PAU-7850_r_tot_ext` FLOAT,
`mag_PAU-7950_r_tot_ext` FLOAT,
`mag_PAU-8050_r_tot_ext` FLOAT,
`mag_PAU-8150_r_tot_ext` FLOAT,
`mag_PAU-8250_r_tot_ext` FLOAT,
`mag_PAU-8350_r_tot_ext` FLOAT,
`mag_PAU-8450_r_tot_ext` FLOAT,
`mag_SDSS-g_r_tot_ext` FLOAT,
`mag_SDSS-i_r_tot_ext` FLOAT,
`mag_SDSS-r_r_tot_ext` FLOAT,
`mag_SDSS-u_r_tot_ext` FLOAT,
`mag_SDSS-z_r_tot_ext` FLOAT,
`mcold` FLOAT,
`mhhalo` FLOAT,
`mstardot` FLOAT,
`mstardot_average` FLOAT,
`mstars_tot` FLOAT,
`r` FLOAT,
`ra` DOUBLE,
`rbulge` FLOAT,
`rdisk` FLOAT,
`type` INT,
`z_cos` FLOAT,
`z_obs` FLOAT
)
STORED AS PARQUET
LOCATION '/user/jcarrete/data/paudm/durham_LC_giorgio/lc5_field1_63'
;

CREATE EXTERNAL TABLE cosmohub.lc5_field1_63_c_bis (
`GalaxyID` BIGINT,
`idrep` INT COMMENT 'ID of box replication this galaxy/halo is found in',
`ra` DOUBLE COMMENT 'right ascension of galaxy (degrees)',
`dec` DOUBLE COMMENT 'declination of galaxy (degrees)',
`r` FLOAT COMMENT 'comoving distance to this galaxy (Mpc/h)',
`z_cos` FLOAT COMMENT 'cosmological redshift of galaxy',
`z_obs` FLOAT COMMENT 'observed redshift of galaxy (peculiar velocities included)',
`type` INT COMMENT 'type/location of galaxy (0=central, 1=satellite in a subhalo, 2=satellite on a particle)',
`BCDM` FLOAT COMMENT 'Band Corrected Distance Modulus = 5log_{10}(d_L/Mpc) + 25 - 2.5log_{10}(1+z)',
`mstardot` FLOAT COMMENT '(instantaneous) quiescent SFR in disk (Msol/h/Gyr)',
`mstardot_average` FLOAT COMMENT 'quiescent SFR in the disk averaged over last timestep (Msol/h/Gyr)',
`mstars_tot` FLOAT COMMENT 'total stellar mass of galaxy (Msol/h) ( = mstars_bulge + mstars_disk )',
`rbulge` FLOAT COMMENT 'half-mass radius of bulge (Mpc/h)',
`rdisk` FLOAT COMMENT 'half-mass radius of disk (Mpc/h)',
`mcold` FLOAT COMMENT 'mass of cold gas in disk of galaxy (Msol/h)',
`mhhalo` FLOAT COMMENT 'mass of the host halo (Msol/h)',
`app_PAU-4550_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-4650_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-4750_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-4850_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-4950_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5050_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5150_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5250_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5350_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5450_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5550_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5650_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5750_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5850_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-5950_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6050_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6150_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6250_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6350_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6450_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6550_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6650_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6750_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6850_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-6950_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7050_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7150_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7250_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7350_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7450_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7550_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7650_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7750_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7850_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-7950_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-8050_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-8150_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-8250_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-8350_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_PAU-8450_o_tot_e` FLOAT COMMENT 'PAU apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_SDSS-g_o_tot_ext` FLOAT COMMENT 'SDSS apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_SDSS-i_o_tot_ext` FLOAT COMMENT 'SDSS apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_SDSS-r_o_tot_ext` FLOAT COMMENT 'SDSS apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_SDSS-u_o_tot_ext` FLOAT COMMENT 'SDSS apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`app_SDSS-z_o_tot_ext` FLOAT COMMENT 'SDSS apparent observed AB magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-4550_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-4650_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-4750_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-4850_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-4950_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5050_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5150_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5250_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5350_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5450_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5550_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5650_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5750_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5850_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-5950_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6050_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6150_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6250_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6350_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6450_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6550_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6650_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6750_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6850_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-6950_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7050_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7150_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7250_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7350_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7450_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7550_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7650_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7750_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7850_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-7950_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-8050_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-8150_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-8250_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-8350_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_PAU-8450_r_tot_ext` FLOAT COMMENT 'PAU absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_SDSS-g_r_tot_ext` FLOAT COMMENT 'SDSS absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_SDSS-i_r_tot_ext` FLOAT COMMENT 'SDSS absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_SDSS-r_r_tot_ext` FLOAT COMMENT 'SDSS absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_SDSS-u_r_tot_ext` FLOAT COMMENT 'SDSS absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)',
`mag_SDSS-z_r_tot_ext` FLOAT COMMENT 'SDSS absolute AB rest frame magnitude (including the contribution of all the components of the galaxy (disk + bulge) and including the internal extinction)'
)
CLUSTERED BY (
GalaxyID)
INTO 4 BUCKETS
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
;

INSERT OVERWRITE TABLE cosmohub.lc5_field1_63_c_bis
SELECT `GalaxyID`, `idrep`, `ra` ,`dec`,`r`, `z_cos`,`z_obs`, `type`, `BCDM`, `mstardot`, `mstardot_average`,`mstars_tot`, `rbulge`, `rdisk`, `mcold`, `mhhalo`, `app_PAU-4550_o_tot_e`, `app_PAU-4650_o_tot_e`, `app_PAU-4750_o_tot_e`, `app_PAU-4850_o_tot_e`, `app_PAU-4950_o_tot_e`, `app_PAU-5050_o_tot_e`, `app_PAU-5150_o_tot_e`, `app_PAU-5250_o_tot_e`, `app_PAU-5350_o_tot_e`, `app_PAU-5450_o_tot_e`, `app_PAU-5550_o_tot_e`, `app_PAU-5650_o_tot_e`, `app_PAU-5750_o_tot_e`, `app_PAU-5850_o_tot_e`, `app_PAU-5950_o_tot_e`, `app_PAU-6050_o_tot_e`, `app_PAU-6150_o_tot_e`, `app_PAU-6250_o_tot_e`, `app_PAU-6350_o_tot_e`, `app_PAU-6450_o_tot_e`, `app_PAU-6550_o_tot_e`, `app_PAU-6650_o_tot_e`, `app_PAU-6750_o_tot_e`, `app_PAU-6850_o_tot_e`,`app_PAU-6950_o_tot_e`,`app_PAU-7050_o_tot_e`,`app_PAU-7150_o_tot_e`, `app_PAU-7250_o_tot_e`, `app_PAU-7350_o_tot_e`, `app_PAU-7450_o_tot_e`, `app_PAU-7550_o_tot_e`, `app_PAU-7650_o_tot_e`, `app_PAU-7750_o_tot_e`, `app_PAU-7850_o_tot_e`, `app_PAU-7950_o_tot_e`, `app_PAU-8050_o_tot_e`, `app_PAU-8150_o_tot_e`, `app_PAU-8250_o_tot_e`, `app_PAU-8350_o_tot_e`, `app_PAU-8450_o_tot_e`,
`app_SDSS-g_o_tot_ext`, `app_SDSS-i_o_tot_ext`, `app_SDSS-r_o_tot_ext`, `app_SDSS-u_o_tot_ext`, `app_SDSS-z_o_tot_ext`, `mag_PAU-4550_r_tot_ext`,
`mag_PAU-4650_r_tot_ext`,
`mag_PAU-4750_r_tot_ext`,
`mag_PAU-4850_r_tot_ext`,
`mag_PAU-4950_r_tot_ext`,
`mag_PAU-5050_r_tot_ext`,
`mag_PAU-5150_r_tot_ext`,
`mag_PAU-5250_r_tot_ext`,
`mag_PAU-5350_r_tot_ext`,
`mag_PAU-5450_r_tot_ext`,
`mag_PAU-5550_r_tot_ext`,
`mag_PAU-5650_r_tot_ext`,
`mag_PAU-5750_r_tot_ext`,
`mag_PAU-5850_r_tot_ext`,
`mag_PAU-5950_r_tot_ext`,
`mag_PAU-6050_r_tot_ext`,
`mag_PAU-6150_r_tot_ext`,
`mag_PAU-6250_r_tot_ext`,
`mag_PAU-6350_r_tot_ext`,
`mag_PAU-6450_r_tot_ext`,
`mag_PAU-6550_r_tot_ext`,
`mag_PAU-6650_r_tot_ext`,
`mag_PAU-6750_r_tot_ext`,
`mag_PAU-6850_r_tot_ext`,
`mag_PAU-6950_r_tot_ext`,
`mag_PAU-7050_r_tot_ext`,
`mag_PAU-7150_r_tot_ext`,
`mag_PAU-7250_r_tot_ext`,
`mag_PAU-7350_r_tot_ext`,
`mag_PAU-7450_r_tot_ext`,
`mag_PAU-7550_r_tot_ext`,
`mag_PAU-7650_r_tot_ext`,
`mag_PAU-7750_r_tot_ext`,
`mag_PAU-7850_r_tot_ext`,
`mag_PAU-7950_r_tot_ext`,
`mag_PAU-8050_r_tot_ext`,
`mag_PAU-8150_r_tot_ext`,
`mag_PAU-8250_r_tot_ext`,
`mag_PAU-8350_r_tot_ext`,
`mag_PAU-8450_r_tot_ext`,
`mag_SDSS-g_r_tot_ext`,
`mag_SDSS-i_r_tot_ext`,
`mag_SDSS-r_r_tot_ext`,
`mag_SDSS-u_r_tot_ext`,
`mag_SDSS-z_r_tot_ext`
FROM jcarrete.lc5_field1_63_pq;

ANALYZE TABLE cosmohub.lc5_field1_63_c_bis COMPUTE STATISTICS;

ANALYZE TABLE cosmohub.lc5_field1_63_c_bis COMPUTE STATISTICS FOR COLUMNS;

Durham-LC-PAU lc5.field1.63 - Revision history

Jcarrete at 20:10, 8 April 2020

Jcarrete: Created page with "I take the data from Giorgio's cephs directory I use some notebook to convert the hdf5 file to parquet file and store it in the /hdfs I use the script to create external tab..."

@@ Line 118: / Line 118: @@
 CREATE EXTERNAL TABLE cosmohub.lc5_field1_63_c_bis (
-     `GalaxyID` BIGINT,
+     `GalaxyID` BIGINT COMMENT 'ID of this galaxy in GALFORM/Millennium Simulation',
      `idrep` INT COMMENT 'ID of box replication this galaxy/halo is found in',
      `ra` DOUBLE COMMENT 'right ascension of galaxy (degrees)',