forked from prjemian/hdf5gateway
-
Notifications
You must be signed in to change notification settings - Fork 1
/
HDF5gateway.ipf
1384 lines (1261 loc) · 47 KB
/
HDF5gateway.ipf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#pragma rtGlobals=3 // Use modern global access method.
#pragma version=1.04
#pragma IgorVersion = 7.00 // 7.00 or later is required for StringFromList offset parameter
// In Igor Pro 8 and before, this requires the Wavemetrics "HDF5.xop" to be installed for IgorPro
// #define DO_HDF5_GATEWAY_TIMING // Define this to turn some timers on for debugging slow performance
//1.04 is modfied by Howard Rodstein on 2021-06-02 to speed up loading data.
// To speed up loading of large files, we started with the hdf5gateway.ipf from // https://github.com/prjemian/hdf5gateway on 2021-05-28.
// modified by HR and JIL to handle liberal file names, but not liberal data names. That will be separate problem for sometimes in the future.
// Some Irena nmodifications needed to be re applied (1.02, 1.03)
//1.03 modified H5GW__make_xref to ship IGORWAVENote which speeds up loading of Irena exprorted data by two order of magnitudes
// modified to import USAXS data as USAXS again. QRS data are all in ImportedData folder, but USAXS is back in USAXS folder.
//1.02 modified H5GW__HDF5AttributeDataToString which failed to read list (dQw,dQl) from resolution attribute.
//1.01 removed KillWaves/Z which took surprisngly long time. Not needed.
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// This file is part of a project hosted at the Advanced Photon Source.
// Access all the source files and data files by checking out the project here:
// git clone https://github.com/prjemian/hdf5gateway.git
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// The documentation is written in restructured text for use by Sphinx.
//
// :doc: http://hdf5gateway.readthedocs.org/
//
// A python program will read this file and grab all the text between lines
// beginning with "//@+" and "//@-" that begin with "//\t" and use them for the Sphinx documentation.
// The tables here have been adjusted for a fixed width font. Don't "fix" them in Igor!
//@+
// ============================================================
// HDF5gateway: HDF5 File I/O Support
// ============================================================
//
// Version: 1.0
//
// HDF5gateway makes it easy to read
// a HDF5 file into an IgorPro folder,
// including group and dataset attributes,
// such as a NeXus data file,
// modify it, and then write it back out.
//
// .. index:: goal
//
// The goal was to make it easy to read a HDF5 file into an IgorPro folder,
// including group and dataset attributes,
// such as a NeXus data file,
// modify it, and then write it back out.
// This file provides functions to do just that.
//
// .. index:: functions; public
//
// Starting with utilities provided in the *HDF5 Browser* package, this file provides
// these public functions:
//
// * :ref:`H5GW_ReadHDF5`
// * :ref:`H5GW_WriteHDF5`
// * :ref:`H5GW_ValidateFolder`
//
// and this function which is useful only for testing and development:
//
// * :ref:`H5GW_TestSuite`
//
// Help is provided with each of these functions to indicate their usage.
//
// .. index:: read
//
// Reading
// ===========
//
// An HDF5 file is read into an IgorPro data folder in these steps:
//
// #. The groups and datasets are read and stored into an IgorPro folder.
// #. Any attributes of these groups and datasets are read and assigned to IgorPro objects.
//
// .. index:: home
//
// The data file is expected to be in the *home* folder (the folder specified by IgorPro's *home* path),
// or relative to that folder, or given by an absolute path name.
//
// .. index:: write, HDF5___xref
//
// Writing
// =================
//
// An IgorPro data folder is written to an HDF5 file in these steps:
//
// #. The IgorPro folder is validated for correct structure.
// #. The objects in the *HDF5___xref* text wave are written to the HDF5 file.
// #. Any folder attributes or wave notes are written to the corresponding HDF5 data path.
//
// The data file is expected to be in the *home* folder (the folder specified by IgorPro's *home* path),
// or relative to that folder, or given by an absolute path name.
//
// .. index:: validate
//
// Validating
// =================
//
// Call :ref:`H5GW_ValidateFolder` to test if the
// *parentFolder* in the IgorPro Data Browser has the proper structure to
// successfully write out to an HDF5 file by :ref:`H5GW_WriteHDF5`.
//
// .. index:: ! HDF5___xref
//
// Structure of the *HDF5___xref* text wave
// =====================================================
//
// It is necessary to devise a method to correlate the name
// of the same object in the HDF5 file with its representation in
// the IgorPro data structure. In IgorPro, certain names are
// reserved such that objects cannot be named. Routines exist
// to substitute such names on data import to comply with
// these restrictions. The routine *HDF5LoadGroup* performs
// this substitution automatically, yet no routine is provided to
// describe any name substitutions performed.
//
// The text wave, *HDF5___xref*, is created in the base folder of
// the IgorPro folder structure to describe the mapping between
// relative IgorPro and HDF5 path names, as shown in the next table.
// This name was chosen in hopes that it might remain unique
// and unused by others at the root level HDF5 files.
//
// HDF5___xref wave column plan
//
// ======= ==================
// column description
// ======= ==================
// 0 HDF5 path
// 1 Igor relative path
// ======= ==================
//
// **Example**
//
// Consider the HDF5 file with datasets stored in this structure:
//
// .. code-block:: guess
// :linenos:
//
// /
// /sasentry01
// /sasdata01
// I
// Q
//
// The next table shows the contents of *HDF5___xref* once this
// HDF5 is read by *H5GW_WriteHDF5()*:
//
// === ======================= ==========================
// row ``HDF5___xref[row][0]`` ``HDF5___xref[row][1]``
// === ======================= ==========================
// 0 / :
// 1 /sasentry01 :sasentry01
// 2 /sasentry01/sasdata01 :sasentry01:sasdata01
// 3 /sasentry01/sasdata01/I :sasentry01:sasdata01:I0
// 4 /sasentry01/sasdata01/Q :sasentry01:sasdata01:Q0
// === ======================= ==========================
//
// Remember, column 0 is for HDF5 paths, column 1 is for IgorPro paths.
//
// On reading an HDF5 file, the *file_name* and *file_path* are written to the
// wave note of *HDF5___xref*. These notations are strictly informative and
// are not used further by this interface. When writing back to HDF5, any
// wave notes of the *HDF5___xref* wave are ignored.
//
// .. rubric:: About *HDF5___xref*:
//
// * Only the folders and waves listed in the *HDF5___xref* text
// wave will be written to the HDF5 file.
// * The *HDF5___xref* text wave is **not written** to the HDF5 file.
//
// When writing an HDF5 file with these functions,
// based on the structure expected in an IgorPro data folder structure,
// the *HDF5___xref* text wave is required. Each IgorPro object described
// must exist as either an IgorPro folder or wave. A wave note is optional.
// For each such IgorPro object, a corresponding HDF5 file object will be created.
//
// .. note:: Important! Any IgorPro data storage objects (folders or waves)
// not listed in *HDF5___xref* **will not be written** to the HDF5 file.
//
// .. index:: group
// .. index:: folder
//
// Groups and Folders
// =====================
//
// An HDF5 *group* corresponds to the IgorPro *folder*. Both are containers
// for either data or containers.
//
// .. index:: Igor___folder_attributes
//
// In HDF5, a group may have attached metadata
// known as *attributes*. In IgorPro, folders have no provision to store
// attributes, thus an optional *Igor___folder_attributes* wave is created. The
// folder attributes are stored in the wave note of this wave. For more information
// about attributes, see the discussion of :ref:`attributes` below.
//
// .. index:: datasets
// .. index:: waves
//
// Datasets and Waves
// ======================
//
// Data is stored in HDF5 datasets and IgorPro waves.
// Both objects are capable of storing a variety of data types
// with different shapes (rank and length). Of the two systems,
// IgorPro is the more restrictive, limiting the rank of stored data
// to four dimensions.
//
// Keep in mind that all components of a single dataset (or wave) are
// of the same data type (such as 64-bit float or 8-bit int).
//
// In HDF5, a dataset may have attached metadata known as
// *attributes*. HDF5 attributes are data structures in their own
// right and may contain data structures. In IgorPro, waves have
// a provision to store attributes in a text construct called the *wave note*.
// Of these two, IgorPro is the more restrictive, unless one creates
// a new wave to hold the data structure of the attributes.
// For more information
// about attributes, see the discussion of :ref:`attributes` below.
//
// The HDF5 library used by this package will take care of converting
// between HDF5 datasets and IgorPro waves and the user need
// not be too concerned about this.
//
//
// .. index:: attributes
// .. index:: ! Igor___folder_attributes
//
// .. _attributes:
//
// Attributes and Wave Notes
// ============================================
//
// Metadata about each of the objects in HDF5 files and IgorPro folders
// is provided by *attributes*. In HDF5, these are attributes directly attached
// to the object (group or dataset). In IgorPro, these attributes are **stored as text** in
// different places depending on the type of the object, as shown in this table:
//
// ======== =======================================================
// object description
// ======== =======================================================
// folder attributes are stored in the wave note of a special
// wave in the folder named *Igor___folder_attributes*
// wave attributes are stored in the wave note of the wave
// ======== =======================================================
//
// .. note:: IgorPro folders do not have a *wave note*
//
// HDF5 allows an attribute to be a data structure with the same rules for
// complexity as a dataset except that attributes must be attached to a dataset
// and cannot themselves have attributes.
//
// .. note:: In IgorPro, attributes will be stored as text.
//
// An IgorPro wave note is a text string that is used here to store a list of
// *key,value* pairs. IgorPro provides helpful routines to manipulate such
// lists, especially when used as wave notes. The IgorPro wave note is the most
// natural representation of an *attribute* except that it does not preserve
// the data structure of an HDF5 attribute without additional coding. This
// limitation is deemed acceptable for this work.
//
// It is most obvious to see
// the conversion of attributes into text by reading and HDF5 file and then
// writing it back out to a new file. The data type of the HDF5 attributes will
// likely be changed from its original type into "string, variable length". If this
// is not acceptable, more work must be done in the routines below.
//
// IgorPro key,value list for the attributes
// ----------------------------------------------------------------------------------------
//
// Attributes are represented in IgorPro wave notes using a
// list of *key,value* pairs. For example:
//
// .. code-block:: guess
// :linenos:
//
// NX_class=SASdata
// Q_indices=0,1
// I_axes=Q,Q
// Mask_indices=0,1
//
// It is important to know the delimiters used by this string to
// differentiate various attributes, some of which may have a
// list of values. Please refer to this table:
//
// =========== ==== ==========================================
// separator char description
// =========== ==== ==========================================
// keySep = between *key* and *value*
// itemSep , between multiple items in *value*
// listSep \\r between multiple *key,value* pairs
// =========== ==== ==========================================
//
// .. note:: A proposition is to store these values in a text wave
// at the base of the folder structure and then use these value
// throughout the folder. This can allow some flexibility with other
// code and to make obvious which terms are used.
//
// .. index:: example
//
// Examples
// ====================
//
// Export data from IgorPro
// -------------------------------------------------------
//
// To write a simple dataset *I(Q)*, one might write this IgorPro code:
//
// .. code-block:: guess
// :linenos:
//
// // create the folder structure
// NewDataFolder/O/S root:mydata
// NewDataFolder/O sasentry
// NewDataFolder/O :sasentry:sasdata
//
// // create the waves
// Make :sasentry:sasdata:I0
// Make :sasentry:sasdata:Q0
//
// Make/N=0 Igor___folder_attributes
// Make/N=0 :sasentry:Igor___folder_attributes
// Make/N=0 :sasentry:sasdata:Igor___folder_attributes
//
// // create the attributes
// Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
// Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
// Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
// Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
// Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
//
// // create the cross-reference mapping
// Make/T/N=(5,2) HDF5___xref
// Edit/K=0 'HDF5___xref';DelayUpdate
// HDF5___xref[0][1] = ":"
// HDF5___xref[1][1] = ":sasentry"
// HDF5___xref[2][1] = ":sasentry:sasdata"
// HDF5___xref[3][1] = ":sasentry:sasdata:I0"
// HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
// HDF5___xref[0][0] = "/"
// HDF5___xref[1][0] = "/sasentry"
// HDF5___xref[2][0] = "/sasentry/sasdata"
// HDF5___xref[3][0] = "/sasentry/sasdata:I"
// HDF5___xref[4][0] = "/sasentry/sasdata:Q"
//
// // Check our work so far.
// // If something prints, there was an error above.
// print H5GW_ValidateFolder("root:mydata")
//
// // set I0 and Q0 to your data
//
// print H5GW_WriteHDF5("root:mydata", "mydata.h5")
//
// .. index:: read
//
// Read data into IgorPro
// -------------------------------------------------------
//
// .. index:: example
//
// This is a simple operation, reading the file from the previous example into a new folder:
//
// .. code-block:: guess
// :linenos:
//
// NewDataFolder/O/S root:newdata
// H5GW_ReadHDF5("", "mydata.h5") // reads into current folder
//@-
// // // // // // // // // // // // // // // // // // // // // // // // // // // // // //
// Return a path possibly quoted for liberal names
static Function/S PossiblyQuotePath(String pathStr)
String sepStr = ":"
pathStr+=sepStr
Variable i ; String rtnStr=""
for (i=0;i<ItemsInList(pathStr,sepStr);i+=1)
rtnStr+=PossiblyQuoteName(StringFromList(i,pathStr,sepStr))+sepStr
endfor
return RemoveEnding(rtnStr,sepStr)
end
static Function/S GetDataFolderPathList(DFREF dfr) // Result is quoted if necessary
String list = ""
String dfPath = GetDataFolder(1, dfr) // dfPath is quoted if necessary
Variable numChildDataFolders = CountObjectsDFR(dfr, 4)
Variable i
for(i=0; i<numChildDataFolders; i+=1)
String childDFName = GetIndexedObjNameDFR(dfr, 4, i)
list += PossiblyQuotePath(dfPath + childDFName) + ";"
DFREF childDFR = dfr:$childDFName
String childList = GetDataFolderPathList(childDFR) // Quoted if necessary
list += childList
endfor
return list
End
//@+
// .. index:: read
//
// Public Functions
// ======================================
//
// .. index:: ! H5GW_ReadHDF5()
//
// .. _H5GW_ReadHDF5:
//
// H5GW_ReadHDF5(parentFolder, fileName, [hdf5Path])
// -------------------------------------------------------------------------------------------------------------
//
// Read the HDF5 data file *fileName* (located in directory *data*,
// an IgorPro path variable) and store it in a subdirectory of
// IgorPro folder *parentFolder*.
//
// At present, the *hdf5Path* parameter is not used. It is planned
// (for the future) to use this to indicate reading only part of the
// HDF5 file to be read.
//
// :String parentFolder: Igor folder path (default is current folder)
// :String fileName: name of file (with extension),
// either relative to current file system directory,
// or include absolute file system path
// :String hdf5Path: path of HDF file to load (default is "/")
// :return String: Status: ""=no error, otherwise, error is described in text
//@-
Function/T H5GW_ReadHDF5(parentFolder, fileName, [hdf5Path])
String parentFolder // If not "", parentFolder must be quoted if necessary
String fileName
String hdf5Path
if ( ParamIsDefault(hdf5Path) )
hdf5Path = "/"
endif
String status = ""
String oldFolder = GetDataFolder(1) // oldFolder is quoted if necessary
parentFolder = H5GW__SetStringDefault(parentFolder, oldFolder)
// First, check that parentFolder exists
if ( DataFolderExists(parentFolder) ) // parentFolder is already quoted if necessary
SetDataFolder $parentFolder
else
return parentFolder + " (Igor folder) not found"
endif
// do the work here:
Variable/G fileID = H5GW__OpenHDF5_RO(fileName)
if ( fileID == 0 )
return fileName + ": could not open as HDF5 file"
endif
// read the data (too bad that HDF5LoadGroup does not read the attributes)
String base_name = StringFromList(0,FileName,".")
base_name = StringFromList(ItemsInList(base_name, ":")-1,base_name,":")
//this is failing on liberal names, so let's use our own name, this should be used to read one file at time and then delete it anyway...
//base_name = "TmpImportNexusFile"
HDF5LoadGroup/Z/L=7/O/R/T=$base_name :, fileID, hdf5Path // recursive
if ( V_Flag != 0 )
SetDataFolder $oldFolder
return fileName + ": problem while opening HDF5 file"
endif
base_name = PossiblyQuoteName(base_name)
// S_objectPaths is quoted if necessary
String wavePaths = S_objectPaths // this gives a clue to renamed datasets (see below for attributes)
String topDFPath = PossiblyQuotePath(parentFolder + ":" + base_name + ":")
DFREF topDFR = $topDFPath
String dataFolderPaths = topDFPath + ";" // e.g., "root:SimpleExampleFile:"
dataFolderPaths += GetDataFolderPathList(topDFR) // Add subdata folders created by HDF5LoadGroup
// Print "Groups:", ItemsInList(dataFolderPaths) // For debugging only
// read the attributes
H5GW__HDF5ReadAttributes(fileID, hdf5Path, base_name, dataFolderPaths, wavePaths)
HDF5CloseFile fileID
String/G file_path
String/G group_name_list
String/G dataset_name_list
// Print "Datasets:", ItemsInList(dataset_name_list) // For debugging only
String xrefPartialPath
sprintf xrefPartialPath, ":%s:HDF5___xref", base_name
xrefPartialPath = PossiblyQuotePath(xrefPartialPath)
WAVE xref = $xrefPartialPath
Note/K xref, "file_name="+fileName
Note xref, "file_path="+file_path
KillStrings/Z file_path, file_name, group_name_list, dataset_name_list
KillVariables/Z fileID
SetDataFolder $oldFolder
return status
End
// ======================================
//@+
// .. index:: write
// .. index:: ! H5GW_WriteHDF5()
//
// .. _H5GW_WriteHDF5:
//
// H5GW_WriteHDF5(parentFolder, newFileName)
// -------------------------------------------------------------------------------------------------------------
//
// Starting with an IgorPro folder constructed such that it passes the :ref:`H5GW_ValidateFolder` test,
// write the components described in *HDF5___xref* to *newFileName*.
//
// :String parentFolder: Igor folder path (default is current folder)
// :String fileName: name of file (with extension),
// either relative to current file system directory,
// or include absolute file system path
//@-
Function/T H5GW_WriteHDF5(parentFolder, newFileName, [replace])
String parentFolder // If not "", parentFolder must be quoted if necessary
String newFileName
Variable replace
if ( ParamIsDefault(replace) )
replace = 1
endif
String status = ""
String oldFolder = GetDataFolder(1) // oldFolder is quoted if necessary
// First, check that parentFolder exists
status = H5GW_ValidateFolder(parentFolder)
if ( strlen(status) > 0 )
return status
endif
SetDataFolder $parentFolder // ???
// Build HDF5 group structure
Variable fileID = H5GW__OpenHDF5_RW(newFileName, replace)
if (fileID == 0)
SetDataFolder $oldFolder
return "Could not create HDF5 file " + newFileName + " for writing"
endif
// write datasets and attributes based on HDF5___xref table
status = H5GW__WriteHDF5_Data(fileID)
if ( strlen(status) > 0 )
HDF5CloseFile fileID
SetDataFolder $oldFolder
return status
endif
HDF5CloseFile fileID
SetDataFolder $oldFolder
return status // report success
End
// ======================================
//@+
// .. index:: validate
// .. index:: ! H5GW_ValidateFolder()
//
// .. _H5GW_ValidateFolder:
//
// H5GW_ValidateFolder(parentFolder)
// -------------------------------------------------------------------------------------------------------------
//
// Check (validate) that a given IgorPro folder has the necessary
// structure for the function H5GW__WriteHDF5_Data(fileID) to be
// successful when writing that folder to an HDF5 file.
//
// :String parentFolder: Igor folder path (default is current folder)
// :return String: Status: ""=no error, otherwise, error is described in text
//@-
Function/T H5GW_ValidateFolder(parentFolder)
String parentFolder // parentFolder must be quoted if necessary
// First, check that parentFolder exists
if ( DataFolderExists(parentFolder) )
SetDataFolder $parentFolder
else
return parentFolder + " (Igor folder) not found"
endif
String oldFolder = GetDataFolder(1) // oldFolder is quoted if necessary
if (1 != Exists("HDF5___xref"))
SetDataFolder $oldFolder
return "required wave (HDF5___xref) is missing in folder: " + parentFolder
endif
Wave/T HDF5___xref
if ( DimSize(HDF5___xref, 1) != 2 )
SetDataFolder $oldFolder
return "text wave HDF5___xref must be of shape (N,2)"
endif
Variable length = DimSize(HDF5___xref, 0), ii
String item, msg
for (ii=0; ii < length; ii=ii+1)
item = HDF5___xref[ii][1]
if ( (1 != DataFolderExists(item)) && (1 != Exists(item)) )
SetDataFolder $oldFolder
return "specified IgorPro object " + item + " was not found in folder " + parentFolder
endif
// TODO: Check that each corresponding HDF5___xref[ii][0] is a valid HDF5 path name
if ( itemsInList(item, ":") != itemsInList(HDF5___xref[ii][0], "/") )
SetDataFolder $oldFolder
msg = "different lengths between HDF5 and IgorPro paths on row" + num2str(ii) + "of HDF5___xref"
return msg
endif
endfor
// TODO: more validation steps
SetDataFolder $oldFolder
return ""
End
// ======================================
//@+
// .. index:: test
// .. index:: ! H5GW_TestSuite()
//
// .. _H5GW_TestSuite:
//
// H5GW_TestSuite()
// -------------------------------------------------------------------------------------------------------------
//
// Test the routines in this file using the supplied test data files.
// HDF5 data files are obtained from the canSAS 2012 repository of
// HDF5 examples
// (http://www.cansas.org/formats/canSAS2012/1.0/doc/_downloads/simpleexamplefile.h5).
//@-
Function H5GW_TestSuite()
String listSep = ";"
String fileExt = ".h5"
String parentDir = "root:worker"
String name_list = "simpleexamplefile"
name_list = name_list +listSep + "simple2dcase"
name_list = name_list +listSep + "simple2dmaskedcase"
name_list = name_list +listSep + "generic2dqtimeseries"
name_list = name_list +listSep + "generic2dtimetpseries"
name_list = name_list +listSep + "NXtest"
Variable length = itemsInList(name_list, listSep), ii
String name, newName, newerName
for (ii = 0; ii < length; ii = ii + 1)
name = StringFromList(ii, name_list, listSep) + fileExt
// Test reading the HDF5 file and then writing the data to a new HDF5 file
newName = H5GW__TestFile(parentDir, name)
// Apply the test again on the new HDF5 file
newerName = H5GW__TestFile(parentDir, newName)
endfor
End
// // // // // // // // // // // // // // // // // // // // // // // // // // // // // //
// ======================================
//@+
// .. index:: functions; private (static)
//
// Private (static) Functions
// ======================================
//
// Documentation of some, but not all, private functions is provided.
//
//@-
//@+
// .. index:: ! H5GW__OpenHDF5_RW()
//
// H5GW__OpenHDF5_RW(newFileName, replace)
// -------------------------------------------------------------------------------------------------------------
//@-
static Function H5GW__OpenHDF5_RW(newFileName, replace)
String newFileName
Variable replace
Variable fileID
if (replace)
HDF5CreateFile/P=home/Z/O fileID as newFileName
else
// make sure file does not exist now, or handle better
HDF5CreateFile/P=home/Z fileID as newFileName
endif
if (V_Flag != 0)
return 0
endif
return fileID
End
//@+
// .. index:: ! H5GW__WriteHDF5_Data()
//
// H5GW__WriteHDF5_Data(fileID)
// -------------------------------------------------------------------------------------------------------------
//@-
static Function/T H5GW__WriteHDF5_Data(fileID)
Variable fileID
String status = ""
Wave/t xref = HDF5___xref
Variable HDF5_col = 0
Variable Igor_col = 1
Variable rows = DimSize(xref,0), ii, groupID, length, jj
String igorPath, hdf5Path, dataType, folder_attr_info, notes, item, key, value
for (ii = 0; ii < rows; ii=ii+1)
igorPath = xref[ii][Igor_col]
hdf5Path = xref[ii][HDF5_col]
// print DataFolderExists(igorPath), igorPath, " --> ", hdf5Path
if ( DataFolderExists(PossiblyQuotePath(igorPath)) )
// group
if ( cmpstr("/", hdf5Path) != 0 )
HDF5CreateGroup /Z fileID , hdf5Path, groupID
if (V_Flag != 0)
status = H5GW__AppendString(status, "\r", "problem creating HDF5 group: " + hdf5Path)
endif
else
groupID = fileID
endif
// attributes
notes = ""
folder_attr_info = H5GW__appendPathDelimiter(igorPath, ":") + "Igor___folder_attributes"
Wave folder_attr = $PossiblyQuotePath(folder_attr_info)
notes = note(folder_attr)
length = itemsInList(notes, "\r")
String response
if ( length > 0 )
for (jj = 0; jj < length; jj=jj+1 )
item = StringFromList(jj, notes,"\r")
key = StringFromList(0, item,"=")
value = StringFromList(1, item,"=")
response = H5GW__SetTextAttributeHDF5(fileID, key, value, hdf5Path)
status = H5GW__AppendString(status, "\r", response)
endfor
endif
else
// dataset
Wave theWave = $PossiblyQuotePath(igorPath)
HDF5SaveData/IGOR=0/Z theWave, fileID, hdf5Path
if (V_Flag != 0)
status = H5GW__AppendString(status, "\r", "problem saving HDF5 dataset: " + hdf5Path)
endif
// look at the wave note for any attributes
notes = note(theWave)
length = itemsInList(notes, "\r")
if ( length > 0 )
for (jj = 0; jj < length; jj=jj+1 )
item = StringFromList(jj, notes,"\r")
key = StringFromList(0, item,"=")
value = StringFromList(1, item,"=")
H5GW__SetTextAttributeHDF5(fileID, key, value, hdf5Path)
endfor
endif
endif
endfor
return status
End
//@+
// .. index:: ! H5GW__SetHDF5ObjectAttributes()
//
// H5GW__SetHDF5ObjectAttributes(itemID, igorPath, hdf5Path)
// -------------------------------------------------------------------------------------------------------------
//@-
static Function H5GW__SetHDF5ObjectAttributes(itemID, igorPath, hdf5Path)
Variable itemID
String igorPath, hdf5Path
Wave theWave = $PossiblyQuotePath(igorPath)
String notes = note(theWave), item, key, value
Variable jj, length = itemsInList(notes, "\r")
notes = note(theWave)
length = itemsInList(notes, "\r")
if ( length > 0 )
for (jj = 0; jj < length; jj=jj+1 )
item = StringFromList(jj, notes,"\r")
key = StringFromList(0, item,"=")
value = StringFromList(1, item,"=")
H5GW__SetTextAttributeHDF5(itemID, key, value, hdf5Path)
endfor
endif
End
//@+
// .. index:: ! H5GW__SetTextAttributeHDF5()
//
// H5GW__SetTextAttributeHDF5(itemID, name, value, hdf5Path)
// -------------------------------------------------------------------------------------------------------------
//@-
static Function/T H5GW__SetTextAttributeHDF5(itemID, name, value, hdf5Path)
Variable itemID
String name, value, hdf5Path
String status = ""
Make/T/N=(1)/Free H5GW____temp
H5GW____temp[0] = value
HDF5SaveData/Z/A=name H5GW____temp, itemID, hdf5Path
if ( V_Flag != 0)
status = "problem saving HDF5 text attribute: " + hdf5Path
endif
//KillWaves H5GW____temp
return status
End
// ======================================
//@+
// .. index:: ! H5GW__make_xref()
//
// H5GW__make_xref(parentFolder, wavePaths, group_name_list, dataset_name_list, base_name)
// ---------------------------------------------------------------------------------------------------------------------------------------------------------
//
// Analyze the mapping between HDF5 objects and Igor paths
// Store the discoveries of this analysis in the HDF5___xref text wave
//
// :String parentFolder: Igor folder path (default is current folder)
// :String wavePaths: Igor paths to loaded waves
// :String group_name_list:
// :String dataset_name_list:
// :String base_name:
//
// HDF5___xref wave column plan
//
// ====== ===============================
// column description
// ====== ===============================
// 0 HDF5 path
// 1 Igor relative path
// ====== ===============================
//@-
static Function H5GW__make_xref(parentFolder, dataFolderPaths, wavePaths, group_name_list, ds_list, base_name)
String parentFolder, dataFolderPaths, wavePaths, group_name_list, ds_list, base_name
#ifdef DO_HDF5_GATEWAY_TIMING
Variable timerRefNum = StartMSTimer
#endif
String xref = "" // key.value pair list as a string
String keySep = "=" // between key and value
String listSep = "\r" // between key,value pairs
String matchStr = PossiblyQuotePath(parentFolder + base_name)
String igorDataFolderPaths = ReplaceString(matchStr, dataFolderPaths, "")
String igorWavePaths = ReplaceString(matchStr, wavePaths, "")
//remove IGORWaveNote stuff, slows everything done and is more or less useless...
igorDataFolderPaths = GrepList(igorDataFolderPaths, "IGORWaveNote" ,1 )
igorWavePaths = GrepList(igorWavePaths, "IGORWaveNote" ,1 )
ds_list = GrepList(ds_list, "IGORWaveNote" ,1 )
// Add data folder/group path pairs
Variable ii, length
length = itemsInList(group_name_list, ";")
int groupOffset = 0
int igorPathOffset = 0
if ( length == itemsInList(igorDataFolderPaths, ";") )
for (ii = 0; ii < length; ii = ii + 1)
String hdf5GroupPath = StringFromList(0, group_name_list, ";", groupOffset)
groupOffset += strlen(hdf5GroupPath) + 1
String igorDFPath = StringFromList(0, igorDataFolderPaths, ";", igorPathOffset)
igorPathOffset += strlen(igorDFPath) + 1
xref = H5GW__addXref(hdf5GroupPath, igorDFPath, xref, keySep, listSep)
endfor
else
// TODO: report an error here and return
endif
// Add wave/dataset path pairs
String dataset, igorPath
// compare items in ds_list and igorWavePaths
length = itemsInList(ds_list, ";")
int datasetOffset = 0
igorPathOffset = 0
if ( length == itemsInList(igorWavePaths, ";") )
for (ii = 0; ii < length; ii = ii + 1)
// ASSUME this is the cross-reference list we need
dataset = StringFromList(0, ds_list, ";", datasetOffset)
datasetOffset += strlen(dataset) + 1
igorPath = StringFromList(0, igorWavePaths, ";", igorPathOffset)
igorPathOffset += strlen(igorPath) + 1
xref = H5GW__addXref(dataset, igorPath, xref, keySep, listSep)
endfor
else
// TODO: report an error here and return
endif
// finally, write the xref contents to a wave
length = itemsInList(xref, listSep)
String file_info
sprintf file_info, ":%s:HDF5___xref", base_name
Make/O/N=(length,2)/T $PossiblyQuotePath(file_info)
Wave/T file_infoT = $PossiblyQuotePath(file_info)
String item
Variable HDF5_col = 0
Variable Igor_col = 1
int xrefOffset = 0
for (ii = 0; ii < length; ii=ii+1)
item = StringFromList(0, xref, listSep, xrefOffset)
xrefOffset += strlen(item) + 1
file_infoT[ii][HDF5_col] = StringFromList(0, item, keySep)
file_infoT[ii][Igor_col] = StringFromList(1, item, keySep)
endfor
#ifdef DO_HDF5_GATEWAY_TIMING
double elapsed = StopMSTimer(timerRefNum) / 1E6
Printf "H5GW__make_xref took %g seconds\r", elapsed
#endif
End
//@+
// .. index:: ! H5GW__addXref()
//
// H5GW__addXref(key, value, xref, keySep, listSep)
// -------------------------------------------------------------------------------------------------------------
//
// append a new key,value pair to the cross-reference list
//@-
static Function/T H5GW__addXref(key, value, xref, keySep, listSep)
String key, value, xref, keySep, listSep
// return xref + key + keySep + value + listSep // Did not help
return H5GW__AppendString(xref, listSep, key + keySep + value)
End
//@+
// .. index:: ! H5GW__appendPathDelimiter()
//
// H5GW__appendPathDelimiter(str, sep)
// -------------------------------------------------------------------------------------------------------------
//@-
static Function/T H5GW__appendPathDelimiter(str, sep)
String str, sep
if ( (strlen(str) == 0) || ( cmpstr(sep, str[strlen(str)-1]) != 0) )
return str + sep
endif
return str
End
// ======================================
//@+
// .. index:: ! H5GW__findTextWaveIndex()
//
// H5GW__findTextWaveIndex(twave, str, col)
// -------------------------------------------------------------------------------------------------------------
//
// :Wave/T twave: correlation between HDF5 and Igor paths
// :String str: text to be located in column *col*
// :int col: column number to search for *str*
// :returns int: index of found text or -1 if not found
//@-
static Function H5GW__findTextWaveIndex(twave, str, col)
Wave/T twave
String str
Variable col
Variable result = -1, ii, rows=DimSize(twave,0)
for (ii=0; ii < rows; ii=ii+1)
if (0 == cmpstr(str, twave[ii][col]) )
result = ii
break
endif
endfor
return result
End
// ======================================
//@+
// .. index:: ! H5GW__OpenHDF5_RO()
//
// H5GW__OpenHDF5_RO(fileName)
// -------------------------------------------------------------------------------------------------------------
//
// :String fileName: name of file (with extension),
// either relative to current file system directory
// or includes absolute file system path
// :returns int: Status: 0 if error, non-zero (fileID) if successful
//
// Assumed Parameter:
//
// * *home* (path): Igor path name (defines a file system
// directory in which to find the data files)
// Note: data is not changed by this function