Legacy Survey DR8 - Revision history

Tallada: /* Join insert */

2020-05-10T12:03:59Z

Join insert

Tallada: /* Sweep photo-z */

2020-05-09T11:05:18Z

Sweep photo-z

Tallada at 12:32, 7 May 2020

2020-05-07T12:32:28Z

Tallada: /* Sweep photo-z */

2020-05-07T09:16:24Z

Sweep photo-z

Show changes

Tallada at 08:51, 7 May 2020

2020-05-07T08:51:04Z

Tallada at 09:18, 28 October 2019

2019-10-28T09:18:55Z

Show changes

Tallada: /* Raw data */

2019-10-25T14:41:20Z

Raw data

Show changes

Tallada: Created page with "== Raw data == Downloaded from ftp://archive.noao.edu:/public/hlsp/ls/dr8/ Directories: south/tractor north/tractor Mirrored using lftp commands: mirror -c -v -P 20..."

2019-10-22T08:01:46Z

Created page with "== Raw data == Downloaded from ftp://archive.noao.edu:/public/hlsp/ls/dr8/ Directories: south/tractor north/tractor Mirrored using lftp commands: mirror -c -v -P 20..."

New page

== Raw data ==

Downloaded from
ftp://archive.noao.edu:/public/hlsp/ls/dr8/

Directories:
south/tractor
north/tractor

Mirrored using lftp commands:
mirror -c -v -P 200 north/tractor/ /mnt/pau/north/tractor/
mirror -c -v -P 200 south/tractor/ /mnt/pau/south/tractor/

into a 5 TiB temporary Ceph filesystem mounted in tdm002.

Checksums provided through sha256sum files, and checked using command:
find * -type d | xargs -t -L1 -P24 sh -c 'cd $1; sha256sum --quiet -c *.sha256sum' cksum

@@ Line 988: / Line 988: @@
 === Join insert ===
-  INSERT OVERWRITE TABLE cosmohub.legacysurvey_dr8_phz
+  INSERT OVERWRITE TABLE cosmohub.legacy_survey_dr8_phz
   SELECT
     dr8.*,

@@ Line 591: / Line 591: @@
 Checksums checked:
-  sha256sum -c legacysurvey_dr8_north_sweep_8.0.sha256sum
+  $ cd north/sweep/8.0
-  sha256sum -c legacysurvey_dr8_south_sweep_8.0.sha256sum
+ $ ls | wc -l
 === Parquet ===

← Older revision		Revision as of 12:32, 7 May 2020
Line 978:		Line 978:
	AND dr8.objid = phz.objid		AND dr8.objid = phz.objid
	;		;
		+
		+	== Queries for checking ==
		+
		+	SELECT COUNT(c), SUM(c)
		+	FROM (
		+	SELECT COUNT(release) AS c
		+	FROM cosmohub.legacy_survey_dr8
		+	GROUP BY release, brickid, objid
		+	) AS t
		+
		+	SELECT COUNT(c), SUM(c)
		+	FROM (
		+	SELECT COUNT(brickid) AS c
		+	FROM cosmohub.legacy_survey_dr8
		+	GROUP BY brickid, objid
		+	) AS t
		+
		+	SELECT COUNT(c), SUM(c)
		+	FROM (
		+	SELECT COUNT(release) AS c
		+	FROM tallada.dr8_sweep
		+	GROUP BY release, brickid, objid
		+	) AS t
		+
		+	SELECT COUNT(c), SUM(c)
		+	FROM (
		+	SELECT COUNT(brickid) AS c
		+	FROM tallada.dr8_sweep
		+	GROUP BY brickid, objid
		+	) AS t
		+
		+	SELECT release, brickid, objid, COUNT(release)
		+	FROM tallada.dr8_sweep
		+	GROUP BY release, brickid, objid
		+	HAVING COUNT(release) > 1
		+
		+	DR8 release brickid objid: DISTINCT(1.694.230.368) TOTAL(1.694.230.368)
		+	DR8 brickid objid: DISTINCT(1.654.429.115) TOTAL(1.694.230.368)
		+
		+	PHZ release brickid objid: DISTINCT(1.646.664.729) TOTAL(1.646.664.897)
		+	PHZ brickid objid: DISTINCT(1.608.841.582) TOTAL(1.646.664.897)
		+
		+	DR8: 1.694.230.368
		+	PHZ: 1.646.664.897
		+	----------------------------------------------------------------------
		+	-47.565.471‬

@@ Line 1: / Line 1: @@
-== Raw data ==
+== Original DR8 ==
 Downloaded from
@@ Line 17: / Line 18: @@
   find * -type d | xargs -t -L1 -P24 sh -c 'cd $1; sha256sum --quiet -c *.sha256sum' cksum
-== Parquet ==
+=== Parquet ===
 The following code snippet was used in spark to convert fits files to parquet:
@@ Line 45: / Line 46: @@
           df.write.parquet(os.path.join(hdfs_base, h, r))
-== External table ==
+=== External table ===
   CREATE EXTERNAL TABLE tallada.dr8 (
@@ Line 217: / Line 218: @@
+  ;
-== Internal Table ==
+=== Internal Table ===
   CREATE TABLE cosmohub.legacy_survey_dr8 (
@@ Line 419: / Line 420: @@
+  ;
-== Comments ==
+=== Comments ===
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE release release smallint COMMENT "Unique integer denoting the camera and filter set used";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE brickid brickid int COMMENT "Brick ID [1,662174]";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE brickname brickname string COMMENT "Name of brick, encoding the brick sky position, eg \"1126p222\" near RA=112.6, Dec=+22.2";
-  ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE objid objid int COMMENT "Catalog object number within this brick; a unique identifier hash is release,brickid,objid; objid spans [0,N-1] and is contiguously enumerated within each brick";
+  ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE objid objid int COMMENT "Catalog object number within this brick\u003b a unique identifier hash is release,brickid,objid\u003b objid spans [0,N-1] and is contiguously enumerated within each brick";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE brick_primary brick_primary boolean COMMENT "True if the object is within the brick boundary";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE brightblob brightblob smallint COMMENT "bitwise mask indicating that an object is near a bright foreground source, as cataloged on the DR8 bitmasks page";
@@ Line 439: / Line 440: @@
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE mjd_max mjd_max double COMMENT "Maximum Modified Julian Date of observations used to construct the model of this object";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE ref_cat ref_cat string COMMENT "Reference catalog source for this star: T2 for Tycho-2, G2 for Gaia DR2, L2 for the LSLGA, empty otherwise";
-  ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE ref_id ref_id bigint COMMENT "Reference catalog identifier for this star; Tyc1*1,000,000+Tyc2*10+Tyc3 for Tycho2; \"sourceid\" for Gaia-DR2 and LSLGA";
+  ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE ref_id ref_id bigint COMMENT "Reference catalog identifier for this star\u003b Tyc1*1,000,000+Tyc2*10+Tyc3 for Tycho2\u003b \"sourceid\" for Gaia-DR2 and LSLGA";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE pmra pmra float COMMENT "Reference catalog proper motion in the RA direction";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE pmdec pmdec float COMMENT "Reference catalog proper motion in the Dec direction";
@@ Line 580: / Line 581: @@
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE shapedev_e2 shapedev_e2 float COMMENT "Ellipticity component 2";
   ALTER TABLE cosmohub.legacy_survey_dr8 CHANGE shapedev_e2_ivar shapedev_e2_ivar float COMMENT "Inverse variance of shapedev_e2";