forked from cubewise-code/tm1py
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathCellService.py
4817 lines (4189 loc) · 229 KB
/
CellService.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-
import asyncio
import csv
import functools
import itertools
import json
import math
import uuid
import warnings
from collections import OrderedDict
from concurrent.futures.thread import ThreadPoolExecutor
from contextlib import suppress
from io import StringIO
from typing import List, Union, Dict, Iterable, Tuple, Optional, Any
import ijson
from mdxpy import MdxHierarchySet, MdxBuilder, Member, MdxTuple
from requests import Response
from TM1py.Exceptions.Exceptions import TM1pyException, TM1pyWritePartialFailureException, TM1pyWriteFailureException, \
TM1pyRestException
from TM1py.Objects.MDXView import MDXView
from TM1py.Objects.NativeView import NativeView
from TM1py.Objects.Process import Process
from TM1py.Objects.Subset import AnonymousSubset
from TM1py.Services.FileService import FileService
from TM1py.Services.ObjectService import ObjectService
from TM1py.Services.ProcessService import ProcessService
from TM1py.Services.RestService import RestService
from TM1py.Services.SandboxService import SandboxService
from TM1py.Services.ViewService import ViewService
from TM1py.Utils import Utils, CaseAndSpaceInsensitiveSet, format_url, add_url_parameters
from TM1py.Utils.Utils import build_pandas_dataframe_from_cellset, dimension_name_from_element_unique_name, \
CaseAndSpaceInsensitiveDict, wrap_in_curly_braces, CaseAndSpaceInsensitiveTuplesDict, \
abbreviate_mdx, build_csv_from_cellset_dict, require_version, require_pandas, build_cellset_from_pandas_dataframe, \
case_and_space_insensitive_equals, get_cube, resembles_mdx, require_data_admin, require_ops_admin, \
extract_compact_json_cellset, \
cell_is_updateable, build_mdx_from_cellset, build_mdx_and_values_from_cellset, \
dimension_names_from_element_unique_names, frame_to_significant_digits, build_dataframe_from_csv, \
drop_dimension_properties, decohints, verify_version
try:
import pandas as pd
_has_pandas = True
except ImportError:
_has_pandas = False
@decohints
def tidy_cellset(func):
""" Higher order function to tidy up cellset after usage
"""
@functools.wraps(func)
def wrapper(self, cellset_id, *args, **kwargs):
try:
return func(self, cellset_id, *args, **kwargs)
finally:
if kwargs.get("delete_cellset", True):
sandbox_name = kwargs.get("sandbox_name", None)
try:
self.delete_cellset(cellset_id=cellset_id, sandbox_name=sandbox_name)
except TM1pyRestException as ex:
# Fail silently if cellset is already removed
if not ex.status_code == 404:
raise ex
return wrapper
@decohints
def manage_transaction_log(func):
""" Control state of transaction log during and after write operation for a given cube through:
`deactivate_transaction_log` and `reactivate_transaction_log`.
Decorated function must have either `cube_name` or `mdx` as first argument or keyword argument
"""
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if "cube_name" in kwargs:
cube_name = kwargs["cube_name"]
elif "mdx" in kwargs:
cube_name = get_cube(kwargs["mdx"])
else:
arg = args[0]
if resembles_mdx(arg):
cube_name = get_cube(arg)
else:
cube_name = arg
deactivate_transaction_log = kwargs.pop("deactivate_transaction_log", False)
reactivate_transaction_log = kwargs.pop("reactivate_transaction_log", False)
try:
if deactivate_transaction_log:
self.deactivate_transactionlog(cube_name)
return func(self, *args, **kwargs)
finally:
if reactivate_transaction_log:
self.activate_transactionlog(cube_name)
return wrapper
@decohints
def manage_changeset(func):
""" Control the start and end of change sets which goups write events together in the TM1 transaction log.
Decorated function working with all non-TI based writing methods
"""
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
use_changeset = kwargs.pop("use_changeset", False)
if use_changeset:
changeset = self.begin_changeset()
try:
return func(self, changeset=changeset, *args, **kwargs)
finally:
self.end_changeset(changeset)
else:
return func(self, *args, **kwargs)
return wrapper
@decohints
def odata_compact_json(return_as_dict: bool):
""" Higher order function to manage header and response when using compact JSON
Applies when decorated function has `use_compact_json` argument set to True
Currently only supports responses with only cell properties and where they are explicitly specified:
* Cellsets('...')?$expand=Axes(...),Cells($select=Ordinal,Value...) does NOT work !
* Cellsets('...')?$expand=Cells does NOT work !
* Cellsets('...')?$expand=Cells($select=Ordinal,Value...) works !
"""
def wrap(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if not kwargs.get("use_compact_json", False):
return func(self, *args, **kwargs)
# Update Accept Header
original_header = self._rest.add_compact_json_header()
try:
response = func(self, *args, **kwargs)
context = response['@odata.context']
if context.startswith('$metadata#Cellsets'):
return extract_compact_json_cellset(context, response, return_as_dict)
else:
raise NotImplementedError('odata_compact_json decorator must only be used on cellsets')
finally:
# Restore original header
self._rest.add_http_header('Accept', original_header)
return wrapper
return wrap
class CellService(ObjectService):
""" Service to handle Read and Write operations to TM1 cubes
"""
def __init__(self, tm1_rest: RestService):
"""
:param tm1_rest: instance of RestService
"""
super().__init__(tm1_rest)
def get_value(self, cube_name: str, elements: Union[str, Iterable] = None, dimensions: List[str] = None,
sandbox_name: str = None, element_separator: str = ",", hierarchy_separator: str = "&&",
hierarchy_element_separator: str = "::", **kwargs) -> Union[str, float]:
""" Returns cube value from specified coordinates
:param cube_name: Name of the cube
:param elements: Describes the Dimension-Hierarchy-Element arrangement
- Example: "Hierarchy1::Element1 && Hierarchy2::Element4, Element9, Element2"
- Dimensions are not specified! They are derived from the position.
- The , separates the element-selections
- If more than one hierarchy is selected per dimension && splits the elementselections
- If no Hierarchy is specified. Default Hierarchy will be addressed
or
Iterable of type mdxpy.Member or similar
- Dimension names must be provided in this case! Example: [(Dimension1, Element1), (Dimension2, Element2), (Dimension3, Element3)]
- Hierarchys can be included. Example: [(Dimension1, Hierarchy1, Element1), (Dimension1, Hierarchy2, Element2), (Dimension2, Element3)]
:param dimensions: List of dimension names in correct order
:param sandbox_name: str
:param element_separator: Alternative separator for the element selections
:param hierarchy_separator: Alternative separator for multiple hierarchies
:param hierarchy_element_separator: Alternative separator between hierarchy name and element name
:return:
"""
mdx_template = "SELECT {} ON ROWS, {} ON COLUMNS FROM [{}]"
mdx_strings_list = []
# Keep backward compatibility with the earlier used "element_string" parameter
if elements is None and "element_string" in kwargs:
elements = kwargs.pop("element_string")
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
# Create MDXpy Member from the element string and get the unique name
# The unique name can be used to build the MDX query directly
if isinstance(elements, str):
element_selections = elements.split(element_separator)
for dimension_name, element_selection in zip(dimensions, element_selections):
if hierarchy_separator not in element_selection:
if hierarchy_element_separator in element_selection:
hierarchy_name, element_name = element_selection.split(hierarchy_element_separator)
else:
hierarchy_name = dimension_name
element_name = element_selection
element_definition = Member.of(dimension_name, hierarchy_name, element_name)
mdx_strings_list.append("{" + element_definition.unique_name + "}")
else:
for element_selection_part in element_selection.split(hierarchy_separator):
hierarchy_name, element_name = element_selection_part.split(hierarchy_element_separator)
element_definition = Member.of(dimension_name, hierarchy_name, element_name)
mdx_strings_list.append("{" + element_definition.unique_name + "}")
else:
# Create MDXpy Member from the Iterator entries
for element_definition in elements:
if not isinstance(element_definition, Member):
element_definition = Member.of(*element_definition)
mdx_strings_list.append("{" + element_definition.unique_name + "}")
# Build the MDX query
# Only the last element is used as the MDX ON COLUMN statement
mdx_rows = "*".join(mdx_strings_list[:-1])
mdx_columns = mdx_strings_list[-1]
mdx = mdx_template.format(mdx_rows, mdx_columns, cube_name)
# Execute MDX
cellset = dict(self.execute_mdx(mdx=mdx, sandbox_name=sandbox_name, **kwargs))
return next(iter(cellset.values()))["Value"]
def get_values(self, cube_name: str, element_sets: Iterable[Iterable[str]] = None, dimensions: List[str] = None,
sandbox_name: str = None, element_separator: str = ",", hierarchy_separator: str = "&&",
hierarchy_element_separator: str = "::", **kwargs) -> List:
""" Returns list of cube values from specified coordinates list. will be in same order as original list
:param cube_name: Name of the cube
:param element_sets: Set of coordinates where each element is provided in the correct dimension order.
[('2024', 'Actual', 'London', 'P02), ('2024', 'Forecast', 'Berlin', 'P03)]
:param dimensions: Dimension names in correct order
:param sandbox_name: str
:param element_separator: Alternative separator for the element selections
:param hierarchy_separator: Alternative separator for multiple hierarchies
:param hierarchy_element_separator: Alternative separator between hierarchy name and element name
:return:
"""
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
q = MdxBuilder.from_cube(cube_name)
for elements in element_sets:
members = []
element_selections = elements.split(element_separator)
for dimension_name, element_selection in zip(dimensions, element_selections):
if hierarchy_separator not in element_selection:
if hierarchy_element_separator in element_selection:
hierarchy_name, element_name = element_selection.split(hierarchy_element_separator)
else:
hierarchy_name = dimension_name
element_name = element_selection
member = Member.of(dimension_name, hierarchy_name, element_name)
members.append(member)
else:
for element_selection_part in element_selection.split(hierarchy_separator):
hierarchy_name, element_name = element_selection_part.split(hierarchy_element_separator)
member = Member.of(dimension_name, hierarchy_name, element_name)
members.append(member)
q.add_member_tuple_to_columns(MdxTuple(members))
# Execute MDX
return self.execute_mdx_values(mdx=q.to_mdx(), sandbox_name=sandbox_name, **kwargs)
def _compose_odata_tuple_from_string(self, cube_name: str,
element_string: str,
dimensions: Iterable[str] = None,
element_separator: str = ",",
hierarchy_separator: str = "&&",
hierarchy_element_separator: str = "::",
**kwargs) -> OrderedDict:
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
odata_tuple_as_dict = OrderedDict()
element_selections = element_string.split(element_separator)
tuple_list = []
for dimension_name, element_selection in zip(dimensions, element_selections):
if hierarchy_separator not in element_selection:
if hierarchy_element_separator in element_selection:
hierarchy_name, element_name = element_selection.split(hierarchy_element_separator)
else:
hierarchy_name = dimension_name
element_name = element_selection
tuple_list.append(format_url("Dimensions('{}')/Hierarchies('{}')/Elements('{}')",
dimension_name,
hierarchy_name,
element_name))
else:
for element_selection_part in element_selection.split(hierarchy_separator):
hierarchy_name, element_name = element_selection_part.split(hierarchy_element_separator)
tuple_list.append(format_url("Dimensions('{}')/Hierarchies('{}')/Elements('{}')",
dimension_name,
hierarchy_name,
element_name))
odata_tuple_as_dict["[email protected]"] = tuple_list
return odata_tuple_as_dict
def _compose_odata_tuple_from_iterable(self, cube_name: str,
element_tuple: Iterable,
dimensions: Iterable[str] = None,
**kwargs) -> OrderedDict:
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
odata_tuple_as_dict = OrderedDict()
odata_tuple_as_dict["[email protected]"] = [
format_url("Dimensions('{}')/Hierarchies('{}')/Elements('{}')", dim, dim, elem)
for dim, elem
in zip(dimensions, element_tuple)]
return odata_tuple_as_dict
def trace_cell_calculation(self, cube_name: str,
elements: Union[Iterable, str],
dimensions: Iterable[str] = None,
sandbox_name: str = None,
depth: int = 1,
element_separator: str = ",",
hierarchy_separator: str = "&&",
hierarchy_element_separator: str = "::",
**kwargs) -> Dict:
""" Trace cell calculation at specified coordinates
:param cube_name: name of the target cube
:param elements:
string "Hierarchy1::Element1 && Hierarchy2::Element4, Element9, Element2"
- Dimensions are not specified! They are derived from the position.
- The , separates the element-selections
- If more than one hierarchy is selected per dimension && splits the elementselections
- If no Hierarchy is specified. Default Hierarchy will be addressed
or
Iterable [Element1, Element2, Element3]
:param dimensions: optional. Dimension names in their natural order. Will speed up the execution!
:param sandbox_name: str
:param depth: optional. Depth of the component trace that will be returned. Deeper traces take longer
:param element_separator: Alternative separator for the elements, if elements are passed as string
:param hierarchy_separator: Alternative separator for multiple hierarchies, if elements are passed as string
:param hierarchy_element_separator: Alternative separator between hierarchy name and element name, if elements are passed as string
:return: trace json string
"""
expand_query = ''
select_query = ''
if depth:
for x in range(1, depth + 1):
component_depth = '/'.join(["Components"] * x)
components_tuple_cube = f'{component_depth}/Tuple($select=Name, UniqueName, Type), {component_depth}/Cube($select=Name)'
expand_query = ','.join([expand_query, components_tuple_cube])
component_fields = f'{component_depth}/Type, {component_depth}/Value, {component_depth}/Statements'
select_query = ','.join([select_query, component_fields])
url = format_url("/Cubes('{}')/tm1.TraceCellCalculation?$select=Type,Value,Statements"
"{}&$expand=Tuple($select=Name, UniqueName, Type) {}", cube_name, select_query, expand_query)
url = add_url_parameters(url, **{"!sandbox": sandbox_name})
if isinstance(elements, str):
body_as_dict = self._compose_odata_tuple_from_string(cube_name,
elements,
dimensions,
element_separator,
hierarchy_separator,
hierarchy_element_separator)
else:
body_as_dict = self._compose_odata_tuple_from_iterable(cube_name, elements, dimensions)
data = json.dumps(body_as_dict, ensure_ascii=False)
return json.loads(self._rest.POST(url=url, data=data, **kwargs).content)
def trace_cell_feeders(self, cube_name: str,
elements: Union[Iterable, str],
dimensions: Iterable[str] = None,
sandbox_name: str = None,
element_separator: str = ",",
hierarchy_separator: str = "&&",
hierarchy_element_separator: str = "::",
**kwargs) -> Dict:
""" Trace feeders from a cell
:param cube_name: name of the target cube
:param elements:
string "Hierarchy1::Element1 && Hierarchy2::Element4, Element9, Element2"
- Dimensions are not specified! They are derived from the position.
- The , separates the element-selections
- If more than one hierarchy is selected per dimension && splits the elementselections
- If no Hierarchy is specified. Default Hierarchy will be addressed
or
Iterable [Element1, Element2, Element3]
:param dimensions: optional. Dimension names in their natural order. Will speed up the execution!
:param sandbox_name: str
:param element_separator: Alternative separator for the elements, if elements are passed as string
:param hierarchy_separator: Alternative separator for multiple hierarchies, if elements are passed as string
:param hierarchy_element_separator: Alternative separator between hierarchy name and element name, if elements are passed as string
:return: feeder trace
"""
url = format_url("/Cubes('{}')/tm1.TraceFeeders?$select=Statements,FedCells"
"&$expand=FedCells/Tuple($select=Name,UniqueName,Type), "
"FedCells/Cube($select=Name)", cube_name)
url = add_url_parameters(url, **{"!sandbox": sandbox_name})
if isinstance(elements, str):
body_as_dict = self._compose_odata_tuple_from_string(cube_name,
elements,
dimensions,
element_separator,
hierarchy_separator,
hierarchy_element_separator)
else:
body_as_dict = self._compose_odata_tuple_from_iterable(cube_name, elements, dimensions)
data = json.dumps(body_as_dict, ensure_ascii=False)
return json.loads(self._rest.POST(url=url, data=data, **kwargs).content)
def check_cell_feeders(self, cube_name: str,
elements: Union[Iterable, str],
dimensions: Iterable[str] = None,
sandbox_name: str = None,
element_separator: str = ",",
hierarchy_separator: str = "&&",
hierarchy_element_separator: str = "::",
**kwargs) -> Dict:
""" Check feeders
:param cube_name: name of the target cube
:param elements:
string "Hierarchy1::Element1 && Hierarchy2::Element4, Element9, Element2"
- Dimensions are not specified! They are derived from the position.
- The , separates the element-selections
- If more than one hierarchy is selected per dimension && splits the elementselections
- If no Hierarchy is specified. Default Hierarchy will be addressed
or
Iterable [Element1, Element2, Element3]
:param dimensions: optional. Dimension names in their natural order. Will speed up the execution!
:param sandbox_name: str
:param element_separator: Alternative separator for the elements, if elements are passed as string
:param hierarchy_separator: Alternative separator for multiple hierarchies, if elements are passed as string
:param hierarchy_element_separator: Alternative separator between hierarchy name and element name, if elements are passed as string
:return: fed cell descriptor
"""
url = format_url("/Cubes('{}')/tm1.CheckFeeders"
"?$select=Fed"
"&$expand=Tuple($select=Name,UniqueName,Type),Cube($select=Name)", cube_name)
url = add_url_parameters(url, **{"!sandbox": sandbox_name})
if isinstance(elements, str):
body_as_dict = self._compose_odata_tuple_from_string(cube_name,
elements,
dimensions,
element_separator,
hierarchy_separator,
hierarchy_element_separator)
else:
body_as_dict = self._compose_odata_tuple_from_iterable(cube_name, elements, dimensions)
data = json.dumps(body_as_dict, ensure_ascii=False)
return json.loads(self._rest.POST(url=url, data=data, **kwargs).content)
def relative_proportional_spread(
self,
value: float,
cube: str,
unique_element_names: Iterable[str],
reference_unique_element_names: Iterable[str],
reference_cube: str = None,
sandbox_name: str = None,
**kwargs) -> Response:
""" Execute relative proportional spread
:param value: value to be spread
:param cube: name of the cube
:param unique_element_names: target cell coordinates as unique element names (e.g. ["[d1].[c1]","[d2].[e3]"])
:param reference_cube: name of the reference cube. Can be None
:param reference_unique_element_names: reference cell coordinates as unique element names
:param sandbox_name: str
:return:
"""
mdx = """
SELECT
{{ {rows} }} ON 0
FROM [{cube}]
""".format(rows="}*{".join(unique_element_names), cube=cube)
cellset_id = self.create_cellset(mdx=mdx, sandbox_name=sandbox_name, **kwargs)
payload = {
"BeginOrdinal": 0,
"Value": "RP" + str(value),
"[email protected]": list(),
format_url("Cubes('{}')", reference_cube if reference_cube else cube)}
for unique_element_name in reference_unique_element_names:
payload["[email protected]"].append(
format_url(
"Dimensions('{}')/Hierarchies('{}')/Elements('{}')",
*Utils.dimension_hierarchy_element_tuple_from_unique_name(unique_element_name)))
return self._post_against_cellset(cellset_id=cellset_id, payload=payload, delete_cellset=True,
sandbox_name=sandbox_name, **kwargs)
def clear_spread(
self,
cube: str,
unique_element_names: Iterable[str],
sandbox_name: str = None,
**kwargs) -> Response:
""" Execute clear spread
:param cube: name of the cube
:param unique_element_names: target cell coordinates as unique element names (e.g. ["[d1].[c1]","[d2].[e3]"])
:param sandbox_name: str
:return:
"""
mdx = """
SELECT
{{ {rows} }} ON 0
FROM [{cube}]
""".format(rows="}*{".join(unique_element_names), cube=cube)
cellset_id = self.create_cellset(mdx=mdx, sandbox_name=sandbox_name, **kwargs)
payload = {
"BeginOrdinal": 0,
"Value": "C",
"[email protected]": list()}
for unique_element_name in unique_element_names:
payload["[email protected]"].append(
format_url(
"Dimensions('{}')/Hierarchies('{}')/Elements('{}')",
*Utils.dimension_hierarchy_element_tuple_from_unique_name(unique_element_name)))
return self._post_against_cellset(cellset_id=cellset_id, payload=payload, delete_cellset=True,
sandbox_name=sandbox_name, **kwargs)
@require_data_admin
@require_ops_admin
@require_version(version="11.7")
def clear(self, cube: str, **kwargs):
"""
Takes the cube name and keyword argument pairs of dimensions and MDX expressions:
```
tm1.cells.clear(
cube="Sales",
salesregion="{[Sales Region].[Australia],[Sales Region].[New Zealand]}",
product="{[Product].[ABC]}",
time="{[Time].[2022].Children}")
```
Make sure that the keyword argument names (e.g. product) map with the dimension names (e.g. Product) in the cube.
Spaces in the dimension name (e.g., "Sales Region") must be omitted in the keyword (e.g. "salesregion")
:param cube: name of the cube
:param kwargs: keyword argument pairs of dimension names and mdx set expressions
:return:
"""
cube_service = self.get_cube_service()
dimension_names = CaseAndSpaceInsensitiveSet(*cube_service.get_dimension_names(cube_name=cube))
dimension_expression_pairs = CaseAndSpaceInsensitiveDict()
for kwarg in kwargs:
if kwarg in dimension_names:
dimension_expression_pairs[kwarg] = wrap_in_curly_braces(kwargs[kwarg])
for dimension_name in dimension_names:
if dimension_name not in dimension_expression_pairs:
expression = MdxHierarchySet.tm1_subset_all(dimension_name).filter_by_level(0).to_mdx()
dimension_expression_pairs[dimension_name] = expression
mdx_builder = MdxBuilder.from_cube(cube).columns_non_empty()
for dimension, expression in dimension_expression_pairs.items():
hierarchy_set = MdxHierarchySet.from_str(dimension=dimension, hierarchy=dimension, mdx=expression)
mdx_builder.add_hierarchy_set_to_column_axis(hierarchy_set)
return self.clear_with_mdx(cube=cube, mdx=mdx_builder.to_mdx(), **kwargs)
@require_data_admin
@require_ops_admin
@require_version(version="11.7")
def clear_with_dataframe(self, cube: str, df: 'pd.DataFrame', dimension_mapping: Dict = None, **kwargs):
"""Clears data from a TM1 cube based on the distinct values in a DataFrame over cube dimensions.
Note:
This function is similar to `tm1.cells.clear`, but it is designed specifically for clearing data
based on distinct values in a DataFrame over cube dimensions. The key difference is that this
function interprets the DataFrame columns as dimensions and supports a mapping (`dimension_mapping`)
for specifying hierarchies within those dimensions.
:param cube: str
The name of the TM1 cube.
:param df: pd.DataFrame
The DataFrame containing distinct values over cube dimensions.
Columns in the DataFrame should correspond to cube dimensions.
:param dimension_mapping: Dict, optional
A dictionary mapping the DataFrame columns to one or many hierarchies within the given dimension.
If not provided, assumes that the dimensions have just one hierarchy.
:return: None
The function clears data in the specified TM1 cube.
:raises ValueError:
If there are unmatched dimensions in the DataFrame or if specified dimensions
do not exist in the TM1 cube.
:example:
```python
# Sample DataFrame with distinct values over cube dimensions
data = {
"Year": ["2021", "2022"],
"Organisation": ["some_company", "some_company"],
"Location": ["Germany", "Albania"]
}
# Sample dimension mapping
dimensions_mapping = {
"Organisation": "hierarchy_1",
"Location": ["hierarchy_2", "hierarchy_3", "hierarchy_4"]
}
dataframe = pd.DataFrame(data)
with TM1Service(**tm1params) as tm1:
tm1.cells.clear_with_dataframe(cube="Sales", df=dataframe)
```
"""
if not dimension_mapping:
dimension_mapping = {}
if len(CaseAndSpaceInsensitiveSet(df.columns)) != len(df.columns):
raise ValueError(f"Column names in DataFrame are not unique identifiers for TM1: {list(df.columns)}")
cube_service = self.get_cube_service()
dimension_names = CaseAndSpaceInsensitiveSet(*cube_service.get_dimension_names(cube_name=cube))
df = df.astype(str)
elements_by_column = {col_name: df[col_name].unique() for col_name in df.columns}
mdx_selections = {}
unmatched_dimension_names = []
for column, elements in elements_by_column.items():
members = []
if column not in dimension_names:
unmatched_dimension_names.append(column)
for element in elements:
if column in dimension_mapping:
hierarchy = dimension_mapping.get(column)
if not isinstance(hierarchy, str):
raise ValueError(f"Value for key '{dimension}' in dimension_mapping must be of type str")
members.append(Member.of(column, hierarchy, element))
else:
members.append(Member.of(column, column, element))
mdx_selections[column] = MdxHierarchySet.members(members)
if dimension_mapping:
for dimension, hierarchies in dimension_mapping.items():
if dimension not in dimension_names:
unmatched_dimension_names.append(dimension)
elif isinstance(hierarchies, str):
hierarchy = hierarchies
mdx_selections[dimension] = MdxHierarchySet.tm1_subset_all(
dimension=dimension,
hierarchy=hierarchy).filter_by_level(0)
elif isinstance(hierarchies, Iterable):
for hierarchy in hierarchies:
mdx_selections[dimension] = MdxHierarchySet.tm1_subset_all(
dimension=dimension,
hierarchy=hierarchy).filter_by_level(0)
else:
raise ValueError(f"Unexpected value type for key '{dimension}' in dimension_mapping")
if unmatched_dimension_names:
raise ValueError(f"Dimension(s) {unmatched_dimension_names} does not exist in cube {cube}."
f"\nCheck the source of the dataframe to fix the problem")
for dimension_name in dimension_names:
if dimension_name not in mdx_selections:
mdx_selections[dimension_name] = MdxHierarchySet.tm1_subset_all(dimension_name).filter_by_level(0)
mdx_builder = MdxBuilder.from_cube(cube).columns_non_empty()
for dimension, expression in mdx_selections.items():
mdx_builder.add_hierarchy_set_to_column_axis(expression)
return self.clear_with_mdx(cube=cube, mdx=mdx_builder.to_mdx(), **kwargs)
@require_data_admin
@require_ops_admin
@require_version(version="11.7")
def clear_with_mdx(self, cube: str, mdx: str, sandbox_name: str = None, **kwargs):
""" clear a slice in a cube based on an MDX query.
Function requires admin permissions, since TM1py uses an unbound TI with a `ViewZeroOut` statement.
:param cube: name of the cube
:param mdx: a valid MDX query
:param sandbox_name: a valid existing sandbox for the current user
:param kwargs:
:return:
"""
view_service = ViewService(self._rest)
enable_sandbox = self.generate_enable_sandbox_ti(sandbox_name)
view_name = "".join(['}TM1py', str(uuid.uuid4())])
view_service.create(MDXView(cube_name=cube, view_name=view_name, MDX=mdx))
try:
code = f"ViewZeroOut('{cube}','{view_name}');"
process = Process(name="")
process.prolog_procedure = enable_sandbox
process.epilog_procedure = code
success, _, _ = self.execute_unbound_process(process, **kwargs)
if not success:
raise TM1pyException(f"Failed to clear cube: '{cube}' with mdx: '{abbreviate_mdx(mdx, 100)}'")
finally:
if view_service.exists(cube, view_name, private=False):
view_service.delete(cube, view_name, private=False)
@tidy_cellset
def _post_against_cellset(self, cellset_id: str, payload: Dict, sandbox_name: str = None, **kwargs) -> Response:
""" Execute a post request against a cellset
:param cellset_id:
:param payload:
:param sandbox_name: str
:param kwargs:
:return:
"""
url = format_url("/Cellsets('{}')/tm1.Update", cellset_id)
url = add_url_parameters(url, **{"!sandbox": sandbox_name})
return self._rest.POST(url=url, data=json.dumps(payload), **kwargs)
def get_dimension_names_for_writing(self, cube_name: str, **kwargs) -> List[str]:
""" Get dimensions of a cube. Skip sandbox dimension
:param cube_name:
:param kwargs:
:return:
"""
from TM1py.Services import CubeService
cube_service = CubeService(self._rest)
dimensions = cube_service.get_dimension_names(cube_name, True, **kwargs)
return dimensions
@require_pandas
def write_dataframe(self, cube_name: str, data: 'pd.DataFrame', dimensions: Iterable[str] = None,
increment: bool = False, deactivate_transaction_log: bool = False,
reactivate_transaction_log: bool = False, sandbox_name: str = None,
use_ti: bool = False, use_blob: bool = False, use_changeset: bool = False,
precision: int = None,
skip_non_updateable: bool = False, measure_dimension_elements: Dict = None,
sum_numeric_duplicates: bool = True, remove_blob: bool = True, allow_spread: bool = False,
clear_view: str = None, static_dimension_elements:Dict = None,
infer_column_order: bool = False,
**kwargs) -> str:
"""
Function expects same shape as `execute_mdx_dataframe` returns.
Column order must match dimensions in the target cube with an additional column for the values.
Column names are not relevant.
:param cube_name:
:param data: Pandas Data Frame
:param dimensions:
:param increment:
:param deactivate_transaction_log:
:param reactivate_transaction_log:
:param sandbox_name:
:param use_ti:
:param use_blob: Uses blob to write. Requires admin permissions. 10x faster compared to use_ti
:param use_changeset: Enable ChangesetID: True or False
:param precision: max precision when writhing through unbound process.
Necessary when dealing with large numbers to avoid "number too long" TI syntax error
:param skip_non_updateable skip cells that are not updateable (e.g. rule derived or consolidated)
:param measure_dimension_elements: dictionary of measure elements and their types to improve
performance when `use_ti` is `True`.
When all written values are numeric you can pass a default dict with default key 'Numeric'
:param sum_numeric_duplicates: Aggregate numerical values for duplicated intersections
:param remove_blob: remove blob file after writing with use_blob=True
:param allow_spread: allow TI process in use_blob or use_ti to use CellPutProportionalSpread on C elements
:param clear_view: name of cube view to clear before writing
:param static_dimension_elements: Dict of fixed dimension element pairs. Column is created for you.
:param infer_column_order: bool indicating whether the column order of the dataframe should automatically be
inferred and mapped to the dimension order in the cube.
:return: changeset or None
"""
if not isinstance(data, pd.DataFrame):
raise ValueError("argument 'data' must of type DataFrame")
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
infer_column_order = True if static_dimension_elements else infer_column_order
if infer_column_order:
dimension_to_column_map = {dim: col for dim in CaseAndSpaceInsensitiveSet(dimensions) for col in data.columns if
col.lower().replace(' ', '') == dim.lower().replace(' ', '')}
column_to_dimension_map = {v: k for k, v in dimension_to_column_map.items()}
# reorder columns in df to align with dimensions; CaseAndSpaceInsensitiveDict is a OrderedDict
if static_dimension_elements:
for dimension, element in static_dimension_elements.items():
if dimension in CaseAndSpaceInsensitiveSet(data.columns):
raise ValueError("one or more of the fixed_dimension_elements are passed as a dataframe column. "
f"{dimension}: {element} is passed in fixed_dimension_elements. "
"Either remove the key value pair from the fixed_dimension_elements dict or "
f"avoid passing the {dimension} column in the dataframe.")
data[dimension] = element
# recreate the maps for infer_column_order if infer_column_order:
if infer_column_order:
dimension_to_column_map = {dim:col for dim in CaseAndSpaceInsensitiveSet(dimensions) for col in data.columns if col.lower().replace(' ', '') == dim.lower().replace(' ', '') }
column_to_dimension_map = {col:dim for col in CaseAndSpaceInsensitiveSet(data.columns) for dim in dimensions if dim.lower().replace(' ', '') == col.lower().replace(' ', '') }
if infer_column_order:
if list(dimension_to_column_map.keys()) != list(column_to_dimension_map.keys()):
# identify the name(s) of the value columns:
columns_not_in_dimensions = [col for col in data.columns if col not in CaseAndSpaceInsensitiveSet(dimensions)]
# get the columns in the cube dimension order with the original column names (CaseAndSpaceInSensitive):
ordered_columns = [dimension_to_column_map[dim] for dim in dimensions if dim in dimension_to_column_map]
# reorder the dataframe:
data = data.loc[:, ordered_columns + columns_not_in_dimensions]
if not len(data.columns) == len(dimensions) + 1:
raise ValueError("Number of columns in 'data' DataFrame must be number of dimensions in cube + 1")
cells = build_cellset_from_pandas_dataframe(data, sum_numeric_duplicates=sum_numeric_duplicates)
return self.write(cube_name=cube_name,
cellset_as_dict=cells,
dimensions=dimensions,
increment=increment,
deactivate_transaction_log=deactivate_transaction_log,
reactivate_transaction_log=reactivate_transaction_log,
sandbox_name=sandbox_name,
use_ti=use_ti,
use_blob=use_blob,
remove_blob=remove_blob,
use_changeset=use_changeset,
precision=precision,
skip_non_updateable=skip_non_updateable,
measure_dimension_elements=measure_dimension_elements,
allow_spread=allow_spread,
clear_view=clear_view,
**kwargs)
@manage_transaction_log
def write_async(self, cube_name: str, cells: Dict, slice_size: int = 250_000, max_workers: int = 8,
dimensions: Iterable[str] = None, increment: bool = False,
deactivate_transaction_log: bool = False, reactivate_transaction_log: bool = False,
sandbox_name: str = None, precision: int = None, measure_dimension_elements: Dict = None,
**kwargs) -> Optional[str]:
""" Write asynchronously
:param cube_name:
:param cells:
:param slice_size:
:param max_workers:
:param dimensions:
:param increment:
:param deactivate_transaction_log:
:param reactivate_transaction_log:
:param sandbox_name:
:param precision: max precision when writhing through unbound process.
Necessary to decrease when dealing with large numbers to avoid "number too long" TI syntax error.
:param measure_dimension_elements: dictionary of measure elements and their types to improve
performance when `use_ti` is `True`.
:param kwargs:
:return:
"""
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
if not measure_dimension_elements:
measure_dimension_elements = self.get_elements_from_all_measure_hierarchies(cube_name=cube_name)
def _chunks(data: Dict):
it = iter(data)
for _ in range(0, len(data), slice_size):
yield {k: data[k] for k in itertools.islice(it, slice_size)}
def _write(chunk: Dict):
return self.write(cube_name=cube_name, cellset_as_dict=chunk, dimensions=dimensions, increment=increment,
use_blob=True, sandbox_name=sandbox_name, precision=precision,
measure_dimension_elements=measure_dimension_elements, **kwargs)
async def _write_async(data: Dict):
loop = asyncio.get_event_loop()
failures = []
with ThreadPoolExecutor(max_workers) as executor:
futures = [loop.run_in_executor(executor, _write, chunk) for chunk in _chunks(data)]
for future in futures:
try:
await future
except (TM1pyWritePartialFailureException, TM1pyWriteFailureException) as exception:
failures.append(exception)
return failures
exceptions = asyncio.run(_write_async(cells))
if not exceptions:
return
# merge all failures into one combined Exception
raise TM1pyWritePartialFailureException(
statuses=list(itertools.chain(*[exception.statuses for exception in exceptions])),
error_log_files=list(itertools.chain(*[exception.error_log_files for exception in exceptions])),
attempts=sum([exception.attempts if isinstance(exception, TM1pyWritePartialFailureException) else 1
for exception in exceptions]))
@require_pandas
@manage_transaction_log
def write_dataframe_async(self, cube_name: str, data: 'pd.DataFrame', slice_size_of_dataframe: int = 250_000,
max_workers: int = 8, dimensions: Iterable[str] = None, increment: bool = False,
sandbox_name: str = None, deactivate_transaction_log: bool = False,
reactivate_transaction_log: bool = False, **kwargs):
""" Write DataFrame into a cube using unbound TI processes in a multi-threading way. Requires admin permissions.
For a DataFrame with > 1,000,000 rows, this function will at least save half of runtime compared with `write_dataframe` function.
Column order must match dimensions in the target cube with an additional column for the values.
Column names are not relevant.
:param cube_name:
:param data: Pandas Data Frame
:param slice_size_of_dataframe: Number of rows for each DataFrame slice, e.g. 10000
:param max_workers: Max number of threads, e.g. 14
:param dimensions:
:param increment: increment or update cell values. Defaults to False.
:param sandbox_name: name of the sandbox or None
:param deactivate_transaction_log:
:param reactivate_transaction_log:
:return: the Future’s result or raise exception.
"""
if not isinstance(data, pd.DataFrame):
raise ValueError("argument 'data' must of type DataFrame")
if not dimensions:
dimensions = self.get_dimension_names_for_writing(cube_name=cube_name)
if not len(data.columns) == len(dimensions) + 1:
raise ValueError("Number of columns in 'data' DataFrame must be number of dimensions in cube + 1")
def _chunks(df: 'pd.DataFrame'):
return [df.iloc[i:i + slice_size_of_dataframe] for i in range(0, df.shape[0], slice_size_of_dataframe)]
def _write(chunk: 'pd.DataFrame'):
return self.write_dataframe(cube_name=cube_name, data=chunk, dimensions=dimensions, increment=increment,
use_blob=True, sandbox_name=sandbox_name, **kwargs)
async def _write_async(df: 'pd.DataFrame'):
loop = asyncio.get_event_loop()
failures = []
with ThreadPoolExecutor(max_workers) as executor:
futures = [loop.run_in_executor(executor, _write, chunk) for chunk in _chunks(df)]