Skip to content

Commit

Permalink
ciss2isis - Set validmax global to input label value (#4589)
Browse files Browse the repository at this point in the history 
* set validmax to inputlabel value

* Converted ciss2isis app and added gtests

* Addressed PR feedback and updated test

* Added test data for ciss2isis tests

Co-authored-by: acpaquette <[email protected]>
  • Loading branch information
@tgiroux @acpaquette
tgiroux and acpaquette authored Feb 7, 2022
1 parent 4763bf4 commit 77cf5d9
Show file tree
Hide file tree
Showing 11 changed files with 1,357 additions and 458 deletions.
385 changes: 385 additions & 0 deletions isis/notebooks/crop_cassiniIss.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,385 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "b6c4ed37",
"metadata": {},
"outputs": [],
"source": [
"import pvl\n",
"import struct\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import datetime\n",
"import os\n",
"import binascii"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "e45c31fe",
"metadata": {},
"outputs": [],
"source": [
"label_file = \"/Users/acpaquette/repos/ISIS3/build/N1472853667_1.LBL\""
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "4eccdc86",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"PVLModule([\n",
" ('PDS_VERSION_ID', 'PDS3')\n",
" ('RECORD_TYPE', 'FIXED_LENGTH')\n",
" ('RECORD_BYTES', 1048)\n",
" ('FILE_RECORDS', 1028)\n",
" ('^IMAGE_HEADER', ['N1472853667_1.IMG', 1])\n",
" ('^TELEMETRY_TABLE', ['N1472853667_1.IMG', 4])\n",
" ('^LINE_PREFIX_TABLE', ['N1472853667_1.IMG', 5])\n",
" ('^IMAGE', ['N1472853667_1.IMG', 5])\n",
" ('ANTIBLOOMING_STATE_FLAG', 'ON')\n",
" ('BIAS_STRIP_MEAN', 17.25049)\n",
" ('CALIBRATION_LAMP_STATE_FLAG', 'N/A')\n",
" ('COMMAND_FILE_NAME', 'trigger_6510_6.ioi')\n",
" ('COMMAND_SEQUENCE_NUMBER', 6510)\n",
" ('DARK_STRIP_MEAN', 0.0)\n",
" ('DATA_CONVERSION_TYPE', 'TABLE')\n",
" ('DATA_SET_ID', 'CO-S-ISSNA/ISSWA-2-EDR-V1.0')\n",
" ('DELAYED_READOUT_FLAG', 'NO')\n",
" ('DESCRIPTION', 'Incomplete product finalized due to truncated lines.')\n",
" ('DETECTOR_TEMPERATURE', Quantity(value=-89.243546, units='DEGC'))\n",
" ('EARTH_RECEIVED_START_TIME',\n",
" datetime.datetime(2004, 9, 3, 9, 54, 37, 232000, tzinfo=datetime.timezone.utc))\n",
" ('EARTH_RECEIVED_STOP_TIME',\n",
" datetime.datetime(2004, 9, 3, 9, 57, 12, 966000, tzinfo=datetime.timezone.utc))\n",
" ('ELECTRONICS_BIAS', 112)\n",
" ('EXPECTED_MAXIMUM', [0.756496, 1.9487])\n",
" ('EXPECTED_PACKETS', 519)\n",
" ('EXPOSURE_DURATION', 220000.0)\n",
" ('FILTER_NAME', ['CL1', 'CL2'])\n",
" ('FILTER_TEMPERATURE', -0.468354)\n",
" ('FLIGHT_SOFTWARE_VERSION_ID', '1.3')\n",
" ('GAIN_MODE_ID', '12 ELECTRONS PER DN')\n",
" ('IMAGE_MID_TIME',\n",
" datetime.datetime(2004, 9, 2, 21, 34, 26, 410000, tzinfo=datetime.timezone.utc))\n",
" ('IMAGE_NUMBER', '1472853667')\n",
" ('IMAGE_OBSERVATION_TYPE', frozenset({'SCIENCE'}))\n",
" ('IMAGE_TIME',\n",
" datetime.datetime(2004, 9, 2, 21, 36, 16, 410000, tzinfo=datetime.timezone.utc))\n",
" ('INSTRUMENT_DATA_RATE', 182.783997)\n",
" ('INSTRUMENT_HOST_NAME', 'CASSINI ORBITER')\n",
" ('INSTRUMENT_ID', 'ISSNA')\n",
" ('INSTRUMENT_MODE_ID', 'FULL')\n",
" ('INSTRUMENT_NAME', 'IMAGING SCIENCE SUBSYSTEM NARROW ANGLE')\n",
" ('INST_CMPRS_PARAM', ['N/A', 'N/A', 'N/A', 'N/A'])\n",
" ('INST_CMPRS_RATE', [3.6, 4.333807])\n",
" ('INST_CMPRS_RATIO', 1.845952)\n",
" ('INST_CMPRS_TYPE', 'LOSSLESS')\n",
" ('LIGHT_FLOOD_STATE_FLAG', 'ON')\n",
" ('METHOD_DESC', 'ISSPT2.5.3;Saturn-Ering;ISS_00ARI_DIFFUSRNG003_PRIME_4')\n",
" ('MISSING_LINES', 511)\n",
" ('MISSING_PACKET_FLAG', 'NO')\n",
" ('MISSION_NAME', 'CASSINI-HUYGENS')\n",
" ('MISSION_PHASE_NAME', 'TOUR PRE-HUYGENS')\n",
" ('OBSERVATION_ID', 'ISS_00ARI_DIFFUSRNG003_PRIME')\n",
" ('OPTICS_TEMPERATURE', [0.712693, 1.905708])\n",
" ('ORDER_NUMBER', 2)\n",
" ('PARALLEL_CLOCK_VOLTAGE_INDEX', 9)\n",
" ('PREPARE_CYCLE_INDEX', 12)\n",
" ('PRODUCT_CREATION_TIME',\n",
" datetime.datetime(2004, 9, 3, 10, 39, 14, tzinfo=datetime.timezone.utc))\n",
" ('PRODUCT_ID', '1_N1472853667.118')\n",
" ('PRODUCT_VERSION_TYPE', 'FINAL')\n",
" ('READOUT_CYCLE_INDEX', 10)\n",
" ('RECEIVED_PACKETS', 576)\n",
" ('SENSOR_HEAD_ELEC_TEMPERATURE', 1.633024)\n",
" ('SEQUENCE_ID', 'S03')\n",
" ('SEQUENCE_NUMBER', 2)\n",
" ('SEQUENCE_TITLE', 'ISS_00ARI_DIFFUSRNG003_PRIME')\n",
" ('SHUTTER_MODE_ID', 'NACONLY')\n",
" ('SHUTTER_STATE_ID', 'ENABLED')\n",
" ('SOFTWARE_VERSION_ID', 'ISS 9.00 02-05-2004')\n",
" ('SPACECRAFT_CLOCK_CNT_PARTITION', 1)\n",
" ('SPACECRAFT_CLOCK_START_COUNT', '1472853447.118')\n",
" ('SPACECRAFT_CLOCK_STOP_COUNT', '1472853667.118')\n",
" ('START_TIME',\n",
" datetime.datetime(2004, 9, 2, 21, 32, 36, 410000, tzinfo=datetime.timezone.utc))\n",
" ('STOP_TIME',\n",
" datetime.datetime(2004, 9, 2, 21, 36, 16, 410000, tzinfo=datetime.timezone.utc))\n",
" ('TARGET_DESC', 'Saturn-Ering')\n",
" ('TARGET_LIST', 'N/A')\n",
" ('TARGET_NAME', 'SATURN')\n",
" ('TELEMETRY_FORMAT_ID', 'UNK')\n",
" ('VALID_MAXIMUM', [9896, 4095])\n",
" ('IMAGE_HEADER',\n",
" {'BYTES': 3144,\n",
" 'HEADER_TYPE': 'VICAR2',\n",
" 'INTERCHANGE_FORMAT': 'ASCII',\n",
" 'RECORDS': 1,\n",
" '^DESCRIPTION': 'VICAR2.TXT'})\n",
" ('TELEMETRY_TABLE',\n",
" {'COLUMN': {'BYTES': 987,\n",
" 'DATA_TYPE': 'MSB_UNSIGNED_INTEGER',\n",
" 'NAME': 'NULL_PADDING',\n",
" 'START_BYTE': 61},\n",
" 'COLUMNS': 2,\n",
" 'INTERCHANGE_FORMAT': 'BINARY',\n",
" 'ROWS': 1,\n",
" 'ROW_BYTES': 1048,\n",
" '^STRUCTURE': 'TLMTAB.FMT'})\n",
" ('LINE_PREFIX_TABLE',\n",
" {'COLUMNS': 7,\n",
" 'INTERCHANGE_FORMAT': 'BINARY',\n",
" 'ROWS': 1024,\n",
" 'ROW_BYTES': 24,\n",
" 'ROW_SUFFIX_BYTES': 1024,\n",
" '^LINE_PREFIX_STRUCTURE': 'PREFIX2.FMT'})\n",
" ('IMAGE',\n",
" {'LINES': 1024,\n",
" 'LINE_PREFIX_BYTES': 24,\n",
" 'LINE_SAMPLES': 1024,\n",
" 'SAMPLE_BITS': 8,\n",
" 'SAMPLE_TYPE': 'SUN_INTEGER'})\n",
"])"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pvl_label = pvl.load(label_file)\n",
"img_file = os.path.join(os.path.split(label_file)[0], pvl_label[\"^IMAGE\"][0])\n",
"pvl_label"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "616e7622",
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"with open(img_file, 'rb') as fp:\n",
" byte_position = ((pvl_label[\"^IMAGE\"][1] - 1) * pvl_label[\"RECORD_BYTES\"])\n",
" b_original_header = fp.read(byte_position)\n",
" b_image_data = fp.read()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "10f0680b",
"metadata": {},
"outputs": [],
"source": [
"lut_list = [1, 3, 5, 7, 9, 11, 14, 16, 19, 22, 25, 28, 31, 35, 38, 42, 46, 49, 53, 57,\n",
"61, 65, 70, 74, 79, 83, 88, 93, 98, 103, 108, 113, 119, 125, 130, 136, 142,\n",
"148, 154, 160, 166, 173, 179, 186, 193, 199, 206, 213, 221, 228, 236, 243,\n",
"251, 259, 267, 275, 283, 291, 299, 307, 316, 324, 333, 342, 351, 360, 369,\n",
"379, 388, 398, 407, 417, 427, 437, 447, 457, 467, 478, 488, 499, 509, 520,\n",
"531, 542, 554, 565, 576, 588, 600, 611, 623, 635, 647, 659, 672, 684, 697,\n",
"709, 722, 735, 747, 761, 774, 787, 801, 814, 828, 842, 856, 870, 884, 898,\n",
"912, 926, 941, 955, 970, 985, 1000, 1015, 1030, 1045, 1061, 1076, 1092, 1108,\n",
"1124, 1140, 1156, 1172, 1188, 1204, 1221, 1237, 1254, 1271, 1288, 1305, 1322,\n",
"1339, 1357, 1374, 1392, 1410, 1428, 1446, 1464, 1482, 1500, 1518, 1537, 1555,\n",
"1574, 1593, 1612, 1631, 1650, 1669, 1689, 1708, 1728, 1748, 1768, 1788, 1808,\n",
"1828, 1848, 1868, 1889, 1909, 1930, 1951, 1971, 1993, 2014, 2035, 2057, 2078,\n",
"2100, 2121, 2143, 2165, 2187, 2209, 2232, 2254, 2276, 2299, 2322, 2344, 2367,\n",
"2390, 2413, 2437, 2460, 2484, 2507, 2531, 2555, 2579, 2603, 2627, 2651, 2676,\n",
"2700, 2725, 2749, 2774, 2799, 2824, 2849, 2874, 2900, 2925, 2951, 2977, 3003,\n",
"3029, 3055, 3081, 3107, 3133, 3160, 3186, 3213, 3239, 3266, 3293, 3321, 3348,\n",
"3375, 3403, 3430, 3458, 3486, 3514, 3542, 3570, 3598, 3627, 3655, 3684, 3712,\n",
"3741, 3770, 3799, 3828, 3857, 3887, 3916, 3946, 3976, 4005, 4035, 4065, 4095]"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "4c26e15b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1048\n"
]
}
],
"source": [
"sampling = (pvl_label['IMAGE']['SAMPLE_BITS']//8)\n",
"line_length = (pvl_label['IMAGE']['LINE_SAMPLES'] * sampling) + pvl_label['IMAGE']['LINE_PREFIX_BYTES']\n",
"print(line_length)\n",
"n_lines = 10\n",
"\n",
"\n",
"image_data = []\n",
"for j in range(n_lines):\n",
" start, stop = j*(line_length), (j+1)*(line_length)\n",
" if (sampling == 2):\n",
" dtype = np.int16\n",
" elif (sampling == 4):\n",
" dtype = np.float32\n",
" else:\n",
" dtype=np.int8\n",
" image_sample = np.frombuffer(b_image_data[start:stop], dtype=dtype, count=int((line_length)/sampling))\n",
" image_data.append(image_sample)\n",
"image_data = np.array(image_data)\n",
"\n",
"# Uncomment the following to compare with an ISIS cube\n",
"# if pvl_label[\"^IMAGE\"][0][0] == 'W':\n",
"# image_data = image_data.byteswap()\n",
"# else:\n",
"# image_data = np.reshape(np.array(list(map(lambda x: lut_list[x], image_data.flatten()))), (n_lines, line_length))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "3815ac94",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4620"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"path_cropped_label = os.path.splitext(label_file)[0] + \".cropped.lbl\"\n",
"path_cropped_file = os.path.splitext(img_file)[0] + \".cropped.img\"\n",
"cropped_file = os.path.split(path_cropped_file)[1]\n",
"\n",
"new_label = pvl_label.copy()\n",
"new_label[\"IMAGE\"][\"LINES\"] = n_lines\n",
"new_label[\"^IMAGE\"][0] = cropped_file\n",
"new_label[\"^IMAGE_HEADER\"][0] = cropped_file\n",
"new_label[\"^TELEMETRY_TABLE\"][0] = cropped_file\n",
"new_label[\"^LINE_PREFIX_TABLE\"][0] = cropped_file\n",
"\n",
"with open(path_cropped_file, 'wb') as fp:\n",
" fp.write(b_original_header)\n",
" fp.write(image_data)\n",
" \n",
"pvl.dump(new_label, path_cropped_label)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28a7937d",
"metadata": {},
"outputs": [],
"source": [
"# Below lines used to compare the ISIS cube to the original label since qt 15.0 doesn't work on big sur\n",
"# as of 01/28/2022"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "31bd22d4",
"metadata": {},
"outputs": [],
"source": [
"cube_file = \"/Path/to/test/output.cub\"\n",
"cube_label = pvl.load(cube_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d3ea86dc",
"metadata": {},
"outputs": [],
"source": [
"with open(cube_file, 'rb') as fp:\n",
" fp.seek(cube_label[\"IsisCube\"][\"Core\"][\"StartByte\"] - 1)\n",
" b_cube_data = fp.read()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b175905",
"metadata": {},
"outputs": [],
"source": [
"line_length = cube_label[\"IsisCube\"][\"Core\"][\"Dimensions\"][\"Samples\"] * 2\n",
"n_lines = cube_label[\"IsisCube\"][\"Core\"][\"Dimensions\"][\"Lines\"]\n",
"\n",
"cube_data = []\n",
"for j in range(n_lines):\n",
" start, stop = j*(line_length), (j+1)*(line_length)\n",
" \n",
" cube_sample = np.frombuffer(b_cube_data[start:stop], dtype=np.int16, count=int(line_length/2))\n",
" cube_data.append(cube_sample)\n",
" \n",
"cube_data = np.array(cube_data)\n",
"# cube_data = np.where(cube_data == -32764, 4095, cube_data)\n",
"cube_data = np.where(cube_data < -32764, 1, cube_data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f8d8c29c",
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"plt.figure(0, figsize=(20, 20))\n",
"plt.imshow(cube_data)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6db691e3",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit 77cf5d9

Please sign in to comment.