From 8bc3206d0c25c461e476d5f719eded31d75129b0 Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 10:40:50 -0400 Subject: [PATCH 01/13] added skip plotting option --- signalgen.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/signalgen.py b/signalgen.py index bd88643..ea187e9 100755 --- a/signalgen.py +++ b/signalgen.py @@ -88,7 +88,8 @@ def addBurst(values): spectra = filterbankObj.get_data(0, samples) #save pre-injection spectra plot - #show_dynamic(spectra, f"{basename} Pre-injection Dynamic Spectra", save=True) + if not skipplot: + show_dynamic(spectra, f"{basename} Pre-injection Dynamic Spectra", save=True) #get bandpass and store in bpWeights bpWeights = create.filter_weights(spectra) @@ -121,9 +122,10 @@ def addBurst(values): gulp = samples ) - logger.info("Saving plot and filterbank.") #and save the new plot - show_dynamic(injectedSpectra, f"{basename} Dynamic Spectra and {values.dm} DM Pulse", save=True) + if not skipplot: + logger.info("Saving plot...") + show_dynamic(injectedSpectra, f"{basename} Dynamic Spectra and {values.dm} DM Pulse", save=True) #now generate new filterbank file newName = f"{basename}_injected.fil" sigprocObj = make_sigproc_object( @@ -148,6 +150,7 @@ def addBurst(values): za_start=-1, ) sigprocObj.write_header(newName) + logger.info("Saving filterbank...") sigprocObj.append_spectra(injectedSpectra, newName) logger.info(f"{newName} successfully written.") @@ -174,11 +177,15 @@ if __name__ == "__main__": parser.add_argument( "-p", "--plot", action="store_true", help="Just plot file and quit." ) + parser.add_argument( + "-s", "--skipplot", action="store_true", help="Skip plotting for large filterbanks." + ) parser.set_defaults(dm=250.0) - parser.set_defaults(nsamp=int(3e4)) + parser.set_defaults(nsamp=int(3e5)) parser.set_defaults(listfile=None) parser.set_defaults(file=None) parser.set_defaults(plot=False) + parser.set_defaults(skipplot=False) values = parser.parse_args() #set working directory to ignored directory From e6903cd188139d6504e1378efd0dc4c84f9523fa Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 10:44:31 -0400 Subject: [PATCH 02/13] oops forgor to check values object --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index ea187e9..a0b41e5 100755 --- a/signalgen.py +++ b/signalgen.py @@ -88,7 +88,7 @@ def addBurst(values): spectra = filterbankObj.get_data(0, samples) #save pre-injection spectra plot - if not skipplot: + if not values.skipplot: show_dynamic(spectra, f"{basename} Pre-injection Dynamic Spectra", save=True) #get bandpass and store in bpWeights @@ -123,7 +123,7 @@ def addBurst(values): ) #and save the new plot - if not skipplot: + if not values.skipplot: logger.info("Saving plot...") show_dynamic(injectedSpectra, f"{basename} Dynamic Spectra and {values.dm} DM Pulse", save=True) #now generate new filterbank file From 14dc9acf57e4bfae9e58f005b90817e31b1f7739 Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 11:03:24 -0400 Subject: [PATCH 03/13] added downsampling to plotter --- signalgen.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/signalgen.py b/signalgen.py index a0b41e5..c613de8 100755 --- a/signalgen.py +++ b/signalgen.py @@ -13,6 +13,7 @@ from scipy.stats import median_abs_deviation from your import Your, Writer from your.formats.filwriter import make_sigproc_object from will import create, inject +from scipy import signal @@ -43,8 +44,13 @@ def show_dynamic( save - Save the plot as `title` + `.png` """ - spectra_mads = median_fitter(np.median(dynamic_spectra, axis=0)) - flat = dynamic_spectra - spectra_mads + + #downsample the spectra first + downFac = max(np.shape(dynamic_spectra)) // 8096 + decSpectra = signal.decimate(dynamic_spectra, downFac, axis=1) + + spectra_mads = median_fitter(np.median(decSpectra, axis=0)) + flat = decSpectra - spectra_mads std = median_abs_deviation(flat, axis=None) med = np.median(flat) plt.figure(figsize=(20, 10)) @@ -198,7 +204,7 @@ if __name__ == "__main__": logging.info(f"Running with file {values.file}") if values.plot: filterbankObj = Your(values.file) - spectra = filterbankObj.get_data(0, 8096) + spectra = filterbankObj.get_data(0, 524288) show_dynamic(spectra, f"{values.file} Dynamic Spectra", save=True) else: addBurst(values) From db2859f6c3a06b19a35c4fe86e237a886e2bdbbd Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 11:11:34 -0400 Subject: [PATCH 04/13] reduced downsampling aggression --- signalgen.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/signalgen.py b/signalgen.py index c613de8..e24d111 100755 --- a/signalgen.py +++ b/signalgen.py @@ -46,9 +46,10 @@ def show_dynamic( """ #downsample the spectra first - downFac = max(np.shape(dynamic_spectra)) // 8096 - decSpectra = signal.decimate(dynamic_spectra, downFac, axis=1) - + downFac = 24 + logging.info(f"Downsampling the axis with min {dynamic_spectra[0,0]} and max {dynamic_spectra[1,-1]}") + decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) + spectra_mads = median_fitter(np.median(decSpectra, axis=0)) flat = decSpectra - spectra_mads std = median_abs_deviation(flat, axis=None) From ae45de8d9b4967253b570feaebc38e7e757f72e4 Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 11:20:17 -0400 Subject: [PATCH 05/13] changed downsamp axis --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index e24d111..c511b90 100755 --- a/signalgen.py +++ b/signalgen.py @@ -47,8 +47,8 @@ def show_dynamic( #downsample the spectra first downFac = 24 - logging.info(f"Downsampling the axis with min {dynamic_spectra[0,0]} and max {dynamic_spectra[1,-1]}") - decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) + logging.info(f"Downsampling the axis with min {dynamic_spectra[2,0]} and max {dynamic_spectra[2,-1]}") + decSpectra = signal.decimate(dynamic_spectra, downFac, axis=2) spectra_mads = median_fitter(np.median(decSpectra, axis=0)) flat = decSpectra - spectra_mads From 120ae8a1c3056ae87d28628d7171274b08e5bf6d Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 11:28:58 -0400 Subject: [PATCH 06/13] nope axis 0 was correct --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index c511b90..99656a4 100755 --- a/signalgen.py +++ b/signalgen.py @@ -47,8 +47,8 @@ def show_dynamic( #downsample the spectra first downFac = 24 - logging.info(f"Downsampling the axis with min {dynamic_spectra[2,0]} and max {dynamic_spectra[2,-1]}") - decSpectra = signal.decimate(dynamic_spectra, downFac, axis=2) + logging.info(f"Downsampling...") + decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) spectra_mads = median_fitter(np.median(decSpectra, axis=0)) flat = decSpectra - spectra_mads From b8e41bac6b01b3304d8acb73e42bcd5ad3170594 Mon Sep 17 00:00:00 2001 From: Abyss Halley Date: Tue, 22 Jul 2025 11:42:33 -0400 Subject: [PATCH 07/13] now with downsampling, try full filterbank object --- signalgen.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/signalgen.py b/signalgen.py index 99656a4..e1c2d4f 100755 --- a/signalgen.py +++ b/signalgen.py @@ -46,9 +46,12 @@ def show_dynamic( """ #downsample the spectra first - downFac = 24 - logging.info(f"Downsampling...") - decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) + downFac = np.shape(dynamic_spectra)[0] // 128000 + if downFac > 1: + logging.info(f"Downsampling...") + decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) + else: + decSpectra = dynamic_spectra spectra_mads = median_fitter(np.median(decSpectra, axis=0)) flat = decSpectra - spectra_mads @@ -85,7 +88,7 @@ def addBurst(values): filterbankObj = Your(values.file) values.file = values.file.strip(".fil") #removing extension for text manipulation later basename = path.basename(values.file) - samples = 524288 + samples = filterbankObj.your_header.nsamp #the full filterbanks use 64GB in RAM when injecting burst, so we write out a truncated version and load that instead. filWriter = Writer(filterbankObj, outdir="./", outname = f"{basename}_trunc", nstart = 0, nsamp = samples) @@ -205,7 +208,7 @@ if __name__ == "__main__": logging.info(f"Running with file {values.file}") if values.plot: filterbankObj = Your(values.file) - spectra = filterbankObj.get_data(0, 524288) + spectra = filterbankObj.get_data(0, filterbankObj.your_header.nsamp) show_dynamic(spectra, f"{values.file} Dynamic Spectra", save=True) else: addBurst(values) From 781767cfb6532c7f1e692a5b28a7ea0c088981a8 Mon Sep 17 00:00:00 2001 From: Sakimori Date: Tue, 22 Jul 2025 11:52:44 -0400 Subject: [PATCH 08/13] use correct header attribute --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index e1c2d4f..5104193 100755 --- a/signalgen.py +++ b/signalgen.py @@ -88,7 +88,7 @@ def addBurst(values): filterbankObj = Your(values.file) values.file = values.file.strip(".fil") #removing extension for text manipulation later basename = path.basename(values.file) - samples = filterbankObj.your_header.nsamp + samples = filterbankObj.your_header.nspectra #the full filterbanks use 64GB in RAM when injecting burst, so we write out a truncated version and load that instead. filWriter = Writer(filterbankObj, outdir="./", outname = f"{basename}_trunc", nstart = 0, nsamp = samples) @@ -208,7 +208,7 @@ if __name__ == "__main__": logging.info(f"Running with file {values.file}") if values.plot: filterbankObj = Your(values.file) - spectra = filterbankObj.get_data(0, filterbankObj.your_header.nsamp) + spectra = filterbankObj.get_data(0, filterbankObj.your_header.nspectra) show_dynamic(spectra, f"{values.file} Dynamic Spectra", save=True) else: addBurst(values) From 15bcdc55c7c2ebca7ad99c0e8f291f49a7ac0ee5 Mon Sep 17 00:00:00 2001 From: Sakimori Date: Tue, 22 Jul 2025 11:59:22 -0400 Subject: [PATCH 09/13] full filterbank didnt work so back to truncated --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index 5104193..e3e6c5b 100755 --- a/signalgen.py +++ b/signalgen.py @@ -88,7 +88,7 @@ def addBurst(values): filterbankObj = Your(values.file) values.file = values.file.strip(".fil") #removing extension for text manipulation later basename = path.basename(values.file) - samples = filterbankObj.your_header.nspectra + samples = 524288 #the full filterbanks use 64GB in RAM when injecting burst, so we write out a truncated version and load that instead. filWriter = Writer(filterbankObj, outdir="./", outname = f"{basename}_trunc", nstart = 0, nsamp = samples) @@ -208,7 +208,7 @@ if __name__ == "__main__": logging.info(f"Running with file {values.file}") if values.plot: filterbankObj = Your(values.file) - spectra = filterbankObj.get_data(0, filterbankObj.your_header.nspectra) + spectra = filterbankObj.get_data(0, 524288) show_dynamic(spectra, f"{values.file} Dynamic Spectra", save=True) else: addBurst(values) From 444d0d5be74befbb580b77a40418a9f84e08e3cc Mon Sep 17 00:00:00 2001 From: Sakimori Date: Tue, 22 Jul 2025 12:07:56 -0400 Subject: [PATCH 10/13] adjusting downfac --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index e3e6c5b..6c1a4b3 100755 --- a/signalgen.py +++ b/signalgen.py @@ -46,9 +46,9 @@ def show_dynamic( """ #downsample the spectra first - downFac = np.shape(dynamic_spectra)[0] // 128000 + downFac = np.shape(dynamic_spectra)[0] // 64000 if downFac > 1: - logging.info(f"Downsampling...") + logging.info(f"Downsampling with factor {downFac}...") decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) else: decSpectra = dynamic_spectra From 8383b821b1a92da8d42a96ae38aa7e4d925d34bc Mon Sep 17 00:00:00 2001 From: Sakimori Date: Tue, 22 Jul 2025 12:23:01 -0400 Subject: [PATCH 11/13] even more aggressive downsampling --- signalgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signalgen.py b/signalgen.py index 6c1a4b3..45d5984 100755 --- a/signalgen.py +++ b/signalgen.py @@ -46,7 +46,7 @@ def show_dynamic( """ #downsample the spectra first - downFac = np.shape(dynamic_spectra)[0] // 64000 + downFac = np.shape(dynamic_spectra)[0] // 8920 if downFac > 1: logging.info(f"Downsampling with factor {downFac}...") decSpectra = signal.decimate(dynamic_spectra, downFac, axis=0) From d1d2573717ddacfc8d455853db5461bfbaf8e534 Mon Sep 17 00:00:00 2001 From: Sakimori Date: Tue, 22 Jul 2025 12:36:57 -0400 Subject: [PATCH 12/13] checking data type int; 0 wasnt correct --- signalgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signalgen.py b/signalgen.py index 45d5984..db898e7 100755 --- a/signalgen.py +++ b/signalgen.py @@ -149,13 +149,13 @@ def addBurst(values): src_raj=filterbankObj.your_header.ra_deg, src_dej=filterbankObj.your_header.dec_deg, machine_id=0, - nbeams=0, + nbeams=1, ibeam=0, nbits=filterbankObj.your_header.nbits, nifs=1, barycentric=0, pulsarcentric=0, - data_type=0, + data_type=1, az_start=-1, za_start=-1, ) From 9ae35a8acaeb34a098b0a2a536896888b3665ac1 Mon Sep 17 00:00:00 2001 From: Sakimori Date: Tue, 22 Jul 2025 13:14:06 -0400 Subject: [PATCH 13/13] fixed beam number index --- signalgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signalgen.py b/signalgen.py index db898e7..3eee25d 100755 --- a/signalgen.py +++ b/signalgen.py @@ -150,7 +150,7 @@ def addBurst(values): src_dej=filterbankObj.your_header.dec_deg, machine_id=0, nbeams=1, - ibeam=0, + ibeam=1, nbits=filterbankObj.your_header.nbits, nifs=1, barycentric=0,