Skip to content

Instantly share code, notes, and snippets.

@maphew

maphew/a-PythonCaller.py

Last active October 17, 2023 20:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save maphew/aa958ad2f2116e292b4e0e3ad2ee771d to your computer and use it in GitHub Desktop.
Save maphew/aa958ad2f2116e292b4e0e3ad2ee771d to your computer and use it in GitHub Desktop.
Download ZIP
A session via chatbotui.com coaching gpt-4 to write and FME PythonCaller function to report on python environment. Elapsed time was ~30-45 minutes. Output html is bit garbage since ChatbotUI doesn't have a share feature yet. The session export json and script to convert is also attached if curious. View rendered html at https://htmlpreview.githu…
Raw
a-PythonCaller.py
import fme
import fmeobjects
import sys
import os
import platform
import pkg_resources
from distutils.sysconfig import get_python_lib
class FeatureProcessor(object):
def __init__(self):
pass
def has_support_for(self, support_type: int):
return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM
def input(self, feature: fmeobjects.FMEFeature):
# Get the environment information
env_info = self.get_environment_info()
# Handling general python attributes
for key, value in env_info.items():
if key != 'library_versions':
new_feature = fmeobjects.FMEFeature()
new_feature.setAttribute("Attribute", key)
new_feature.setAttribute("Value", str(value))
self.pyoutput(new_feature)
# Handling library versions separately
for library, version in env_info['library_versions'].items():
new_feature = fmeobjects.FMEFeature()
new_feature.setAttribute("Attribute", library)
new_feature.setAttribute("Value", version)
self.pyoutput(new_feature)
def close(self):
pass
def process_group(self):
pass
def get_environment_info(self):
"""Outputs a list of the currently active python environment, location on disk, and version numbers."""
env_info = {}
# Getting Python Version and location
env_info["python_version"] = platform.python_version()
env_info["python_location"] = sys.executable
# Getting the local site-packages path
env_info["site_packages_path"] = get_python_lib()
# Getting installed libraries
installed_libraries = pkg_resources.working_set
library_versions = {library.key: library.version for library in installed_libraries}
env_info["library_versions"] = library_versions
return env_info
Raw
chbotui-session2html.py
import json
import os
import sys
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter
# Load json data
data = """
{
"folderId": null,
"id": "f3b60060-cab1-4135-b007-6c7494be68e5",
"messages": [
{
"content": "Below is the default code for PythonCaller. Add a function ",
"role": "user"
},
{
"content": "Sure, here's an example of that Python code function within",
"role": "assistant"
},
{
"content": "The code runs successfully but FME reports empty output. He",
"role": "user"
},
{
"content": "The issue refers that the attributes of a specific feature ",
"role": "assistant"
},
{
"content": "Ah, success. Now it works.",
"role": "user"
},
{
"content": "Great to hear that it's working now!\\n\\nJust for the sake o",
"role": "assistant"
}
],
"model": {
"id": "gpt-4",
"name": "GPT-4"
},
"name": "FME: report python environment details",
"prompt": "You are a large language model with a special focus on Feature Manipulation Engine (FME) and adept and helping users generate code for use in it's Regex, SQL, and Python editors.",
"temperature": 1
}
"""
# If a command line argument is provided, use it as a filename to read json data
if len(sys.argv) > 1:
with open(sys.argv[1], 'r') as json_file:
data = json.load(json_file)
# If command line argument is not provided, try to read from input.json file
elif os.path.exists('input.json'):
with open('input.json','r') as json_file:
data = json.load(json_file)
# If both are not provided or do not exist, use internal data
else:
data = json.loads(internal_data)
# Format data as html
html = """
<html>
<head>
<title>{title}</title>
<style>
body {{
font-family: Arial, sans-serif;
}}
.user {{
color: blue;
font-weight: bold;
}}
.assistant {{
color: green;
}}
</style>
</head>
<body>
<h1>{title}</h1>
<p>{prompt}</p>
""".format(title=data['name'], prompt=data['prompt'])
# Before the 'for' loop that iterates over messages
formatter = HtmlFormatter()
for message in data['messages']:
if '```python' in message['content'] and '```' in message['content']:
content = message['content'].split("```python")[1].split("```")[0]
highlighted_content = highlight(content, PythonLexer(), formatter)
message['content'] = message['content'].replace("```python" + content + "```", highlighted_content)
html += """
<p class="{role}">{content}</p>
""".format(role=message['role'], content=message['content'])
html += "<style>\n"
html += formatter.get_style_defs('.highlight')
html += "\n</style>"
for message in data['messages']:
html += """
<p class="{role}">{content}</p>
""".format(role=message['role'], content=message['content'])
html += """
</body>
</html>
"""
# Write html to file
with open('output.html', 'w') as f:
f.write(html)
Raw
session-output.html
<html>
<head>
<title>FME: report python environment details</title>
<style>
body {
font-family: Arial, sans-serif;
}
.user {
color: blue;
font-weight: bold;
}
.assistant {
color: green;
}
</style>
</head>
<body>
<h1>FME: report python environment details</h1>
<p>You are a large language model with a special focus on Feature Manipulation Engine (FME) and adept and helping users generate code for use in it's Regex, SQL, and Python editors.</p>
<p class="user">Below is the default code for PythonCaller. Add a function that outputs a list of the currently active python environment, location on disk, and version numbers of primary modules and libraries important for use in FME.
import fme
import fmeobjects
class FeatureProcessor(object):
"""Template Class Interface:
When using this class, make sure its name is set as the value of the 'Class to Process Features'
transformer parameter.
"""
def __init__(self):
"""Base constructor for class members."""
pass
def has_support_for(self, support_type: int):
"""This method is called by FME to determine if the PythonCaller supports Bulk mode,
which allows for significant performance gains when processing large numbers of features.
Bulk mode cannot always be supported.
More information available in transformer help.
"""
return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM
def input(self, feature: fmeobjects.FMEFeature):
"""This method is called for each feature which enters the PythonCaller.
Processed input features can be emitted from this method using self.pyoutput().
If knowledge of all input features is required for processing, then input features should be
cached to a list instance variable and processed using group processing or in the close() method.
"""
self.pyoutput(feature)
def close(self):
"""This method is called once all the FME Features have been processed from input()."""
pass
def process_group(self):
"""This method is called by FME for each group when group processing mode is enabled.
This implementation should reset any instance variables used for the next group.
Bulk mode should not be enabled when using group processing.
More information available in transformer help.
"""
pass</p>
<p class="assistant">Sure, here's an example of that Python code function within your class. I've used `sys`, `os`, `platform`, `pip` and `importlib` modules to get the required information.
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">fme</span>
<span class="kn">import</span> <span class="nn">fmeobjects</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">pip</span>
<span class="kn">import</span> <span class="nn">importlib</span>
<span class="c1"># Necessary to get the local site-packages path</span>
<span class="kn">from</span> <span class="nn">distutils.sysconfig</span> <span class="kn">import</span> <span class="n">get_python_lib</span>
<span class="k">class</span> <span class="nc">FeatureProcessor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">has_support_for</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">support_type</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">return</span> <span class="n">support_type</span> <span class="o">==</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FME_SUPPORT_FEATURE_TABLE_SHIM</span>
<span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">feature</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">process_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Outputs a list of the currently active python environment, location on disk, and version numbers.&quot;&quot;&quot;</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Getting Python Version and location</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="c1"># Getting the local site-packages path</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="c1"># Getting installed libraries</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pip</span><span class="o">.</span><span class="n">get_installed_distributions</span><span class="p">()</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
Note: This function (`get_environment_info`) would return a dictionary with keys "python_version", "python_location", "site_packages_path", and "library_versions". The "library_versions" is itself a dictionary with keys as the library names and values as their respective versions. You can call this method within `input` method to see the details for each feature.
Please note, this code was written for Python 2 and may need to be adjusted if you are working with Python 3, as some libraries locations are handled differently between versions. Specifically, from Python 3.8 onwards, the `pip.get_installed_distributions()` method has been removed. You could use `pip freeze` command and parse its output, or use a library like `pkg_resources`.</p>
<p class="user">The code runs successfully but FME reports empty output. Here is the relevant log secton:
PythonCreator_Creator (CreationFactory): Created 1 features
Optional `process_group' method not present; not called
PythonCreator_OUTPUT Brancher -1 4 (BranchingFactory): Branched 1 input feature -- 1 feature routed to the target factory, and 0 features routed to the fallback factory.
_CREATOR_BRANCH_TARGET (TeeFactory): Cloned 1 input feature(s) into 1 output feature(s)
Destination Feature Type Routing Correlator (RoutingFactory): Tested 0 input feature(s), wrote 0 output feature(s): 0 matched merge filters, 0 were routed to output, 0 could not be routed.
Final Output Nuker (TeeFactory): Cloned 0 input feature(s) into 0 output feature(s)
</p>
<p class="assistant">It seems like there's been some confusion. The function `get_environment_info` is merely returning the dictionary of the information about the environment, but it isn't outputting this data as a feature nor writing it into the logs.
To output this data as a feature, you would need to call `get_environment_info` within `input` or `close` methods and utilize `fmeobjects.FMEFeature` to transform this data into a format FME can handle.
Here's a possible way to do this with making use of your `input` method:
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">fme</span>
<span class="kn">import</span> <span class="nn">fmeobjects</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">pip</span>
<span class="kn">import</span> <span class="nn">importlib</span>
<span class="kn">from</span> <span class="nn">distutils.sysconfig</span> <span class="kn">import</span> <span class="n">get_python_lib</span>
<span class="k">class</span> <span class="nc">FeatureProcessor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">has_support_for</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">support_type</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">return</span> <span class="n">support_type</span> <span class="o">==</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FME_SUPPORT_FEATURE_TABLE_SHIM</span>
<span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="c1"># Call get_environment_info instead of self.pyoutput(feature)</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_environment_info</span><span class="p">()</span>
<span class="c1"># Use the dictionary to create a new feature</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">env_info</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">process_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pip</span><span class="o">.</span><span class="n">get_installed_distributions</span><span class="p">()</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
This will output each piece of information about the environment as a separate feature. In the `input` method, I've converted the `env_info` dictionary to a series of FME features which are output via `self.pyoutput`. Each resulting feature has one attribute, with the key and the associated value.
Note that values that are dictionaries or lists need to be converted to string via `str` to be compatible with `FMEFeature`. Hence, the library versions will appear as a string representation of a dictionary.
With these changes, you should be able to start seeing output features in your workspace.</p>
<p class="user">Python Exception <AttributeError>: module 'pip' has no attribute 'get_installed_distributions'
</p>
<p class="assistant">The method `pip.get_installed_distributions()` was removed starting with pip version 10.0. Instead, you could use `pkg_resources` to get a list of installed distributions. Here's your updated `get_environment_info` method:
<div class="highlight"><pre><span></span> <span class="kn">import</span> <span class="nn">pkg_resources</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Outputs a list of the currently active python environment, location on disk, and version numbers.&quot;&quot;&quot;</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Getting Python Version and location</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="c1"># Getting the local site-packages path</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="c1"># Getting installed libraries</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">working_set</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
This will get you the same info as the previous version. Note that `pkg_resources.working_set` is a list of all installed distributions.
</p>
<p class="user">what needs to be entered in to the "Attributes to Expose" field of the "Output Attributes" section of the PythonCaller dialog to use the values returned by the code?</p>
<p class="assistant">The "Attributes to Expose" field specifies which Python variables you want to expose as attributes in the FME feature schema. These exposed attributes will be viewable and usable within your FME workspace.
Since you are outputting attributes dynamically by setting them directly on the FMEFeature object within the `input` method, you do not need to specify anything in the 'Attributes to Expose' field. The created attributes ("python_version", "python_location", "site_packages_path" and "library_versions") will be added to the feature regardless of what is specified in the 'Attributes to Expose' field.
If you were returning a dictionary from the `input` method, you would need to specify the keys of the dictionary in the 'Attributes to Expose' field. But since you are creating attributes directly on the features by using `setAttribute`, this is not necessary.
However, if you still want to explicity expose attribute via "Attributes to Expose" field, you could enter a comma-separated list like so:
```
python_version, python_location, site_packages_path, library_versions
```
Ensure these align exactly with the keys in the dictionary returned by `get_environment_info`, including case sensitivity.</p>
<p class="user">When I leave Attributes to Expose empty I get output of 4 rows and 1 column, all values empty.
When I enter `python_version, python_location, site_packages_path, library_versions` I get the output in Visual Preview. (A 4 by 4 table matrix, white space delimited, all cells empty except the bottom right, corresponding to col `library_versions` and row 3):
```
python_version, python_location, site_packages_path, library_versions
{'fme-emailer': '3.3.3', 'certifi': '2022.9.24', 'regex': '2022.7.9', 'fsspec': '2022.7.1', 'dask': '2022.7.0', 'distributed': '2022.7.0', 'pefile': '2022.5.30', 'setuptools': '65.5.1', 'cryptography': '38.0.4', 'pyzmq': '24.0.1', 'keyring': '23.4.0', 'pip': '22.2.2', 'pyopenssl': '22.0.0', 'attrs': '21.4.0', 'argon2-cffi': '21.3.0', 'packaging': '21.3', 'argon2-cffi-bindings': '21.2.0', 'azure-storage-blob': '12.8.0', 'pillow': '9.5.0', 'click': '8.0.4', 'ipython': '7.34.0', 'ipywidgets': '7.7.2', 'nbconvert': '7.2.1', 'pytest': '7.2.0', 'ipykernel': '6.16.0', 'notebook': '6.4.12', 'jupyter-console': '6.4.2', 'jupyter-client': '6.1.12', 'tornado': '6.1', 'pyyaml': '6.0', 'psutil': '5.9.0', 'nbformat': '5.6.1', 'traitlets': '5.5.0', 'ujson': '5.4.0', 'decorator': '5.1.1', 'tqdm': '4.64.1', 'fonttools': '4.25.0', 'jsonschema': '4.16.0', 'importlib-metadata': '4.11.3', 'jupyter-core': '4.11.2', 'beautifulsoup4': '4.11.1', 'lxml': '4.9.1', 'saspy': '4.3.2', 'typing-extensions': '4.3.0', 'cachetools': '4.2.2', 'bleach': '4.1.0', 'pyodbc': '4.0.34', 'flake8': '4.0.1', 'chardet': '4.0.0', 'lerc': '4.0', 'protobuf': '3.19.4', 'zipp': '3.8.0', 'h5py': '3.7.0', 'widgetsnbextension': '3.6.1', 'matplotlib': '3.6.0', 'anyio': '3.5.0', 'jupyterlab': '3.4.8', 'idna': '3.4', 'oauthlib': '3.2.0', 'lz4': '3.1.3', 'pro-notebook-integration': '3.1', 'openpyxl': '3.0.10', 'pyparsing': '3.0.9', 'prompt-toolkit': '3.0.5', 'jinja2': '3.0.3', 'jupyterlab-widgets': '3.0.3', 'requests': '2.28.1', 'pycparser': '2.21', 'fastjsonschema': '2.16.2', 'jupyterlab-server': '2.16.0', 'pygments': '2.9.0', 'networkx': '2.8.4', 'python-dateutil': '2.8.2', 'numexpr': '2.8.1', 'pycodestyle': '2.8.0', 'platformdirs': '2.5.2', 'affine': '2.4.0', 'pyflakes': '2.4.0', 'pyjwt': '2.4.0', 'sortedcontainers': '2.4.0', 'soupsieve': '2.3.2.post1', 'mkl-service': '2.3.0', 'pyshp': '2.1.3', 'markupsafe': '2.1.1', 'arcgis': '2.1.0.2', 'zict': '2.1.0', 'asttokens': '2.0.5', 'charset-normalizer': '2.0.4', 'mistune': '2.0.4', 'pywinpty': '2.0.1', 'shapely': '2.0.1', 'tomli': '2.0.1', 'xlrd': '2.0.1', 'cloudpickle': '2.0.0', 'botocore': '1.29.76', 'boto3': '1.26.76', 'urllib3': '1.26.12', 'pypdf2': '1.26.0', 'jupyter-server': '1.21.0', 'numpy': '1.20.1', 'six': '1.16.0', 'cffi': '1.15.1', 'wrapt': '1.14.1', 'azure-core': '1.12.0', 'swat': '1.12.0', 'gssapi': '1.8.1', 'send2trash': '1.8.0', 'pysocks': '1.7.1', 'tblib': '1.7.0', 'python-certifi-win32': '1.7.dev1+g24a5d1b', 'cftime': '1.6.2', 'scipy': '1.6.2', 'netcdf4': '1.6.1', 'pystac': '1.6.1', 'nest-asyncio': '1.5.5', 'debugpy': '1.5.1', 'pandocfilters': '1.5.0', 'sqlalchemy': '1.4.39', 'snuggs': '1.4.7', 'jupyter-latex-envs': '1.4.6', 'appdirs': '1.4.4', 'pandas': '1.4.4', 'typed-ast': '1.4.3', 'kiwisolver': '1.4.2', 'jdcal': '1.4.1', 'blinker': '1.4', 'ntlm-auth': '1.4.0', 'nose': '1.3.7', 'rasterio': '1.3.6', 'bottleneck': '1.3.5', 'mkl-fft': '1.3.0', 'xlwt': '1.3.0', 'requests-oauthlib': '1.3.0', 'requests-gssapi': '1.2.3', 'mkl-random': '1.2.2', 'mpmath': '1.2.1', 'partd': '1.2.0', 'sniffio': '1.2.0', 'munkres': '1.1.4', 'click-plugins': '1.1.1', 'greenlet': '1.1.1', 'iniconfig': '1.1.1', 'tinycss2': '1.1.1', 'et-xmlfile': '1.1.0', 'win-inet-pton': '1.1.0', 'contourpy': '1.0.5', 'msgpack': '1.0.3', 'heapdict': '1.0.1', 'jmespath': '1.0.1', 'geomet': '1.0.0', 'locket': '1.0.0', 'exceptiongroup': '1.0.0rc9', 'websocket-client': '0.58.0', 'olefile': '0.46', 'wheel': '0.37.1', 'xarray': '0.20.1', 'pyrsistent': '0.18.0', 'pluggy': '0.13.1', 'terminado': '0.13.1', 'seaborn': '0.12.1', 'requests-kerberos': '0.12.0', 'toolz': '0.11.2', 'cytoolz': '0.11.0', 'cycler': '0.11.0', 'toml': '0.10.2', 'json5': '0.9.5', 'requests-toolbelt': '0.9.1', 'pathspec': '0.9.0', 'executing': '0.8.3', 'parso': '0.8.3', 'prometheus-client': '0.8.0', 'winkerberos': '0.8.0', 'pickleshare': '0.7.5', 'cligj': '0.7.2', 'defusedxml': '0.7.1', 'brotlipy': '0.7.0', 'mccabe': '0.7.0', 'nbclient': '0.7.0', 'msrest': '0.6.21', 'isodate': '0.6.0', 'jupyter-contrib-nbextensions': '0.6.0', 's3transfer': '0.6.0', 'webencodings': '0.5.1', 'stack-data': '0.5.1', 'testpath': '0.5.0', 'nbclassic': '0.4.8', 'colorama': '0.4.5', 'mypy-extensions': '0.4.3', 'jupyter-nbextensions-configurator': '0.4.1', 'x86cpu': '0.4', 'entrypoints': '0.4', 'jupyter-contrib-core': '0.4.0', 'wcwidth': '0.2.5', 'jupyterlab-pygments': '0.2.2', 'pure-eval': '0.2.2', 'pywin32-ctypes': '0.2.0', 'wincertstore': '0.2', 'backcall': '0.2.0', 'despatch': '0.2.0', 'ipython-genutils': '0.2.0', 'jupyter-highlight-selected-word': '0.2.0', 'matplotlib-inline': '0.1.6', 'notebook-shim': '0.1.0', 'black': '0.0.0', 'requests-negotiate-sspi': '0.0.0', 'babel': '2.11.0', 'future': '0.18.2', 'jedi': '0.18.2', 'pytz': '2022.6', 'sympy': '1.9', 'filelock': '3.10.6', 'tldextract': '3.4.0', 'markdown': '3.1.1', 'xlsxwriter': '3.0.9', 'tzlocal': '2.1', 'hdfs': '2.0.16', 'diskcache': '2.0.2', 'requests-file': '1.5.1', 'requests-aws4auth': '1.2.2', 'requests-ntlm': '1.2.0', 'pypac': '0.16.1', 'xmltodict': '0.13.0', 'python-docx': '0.8.11', 'pyspnego': '0.8.0', 'python-pptx': '0.6.21', 'docopt': '0.6.2', 'dukpy': '0.3.0'}
```
</p>
<p class="assistant">This error suggests that the loop handling creating new features and setting attributes is not working correctly. The result "`library_versions`" should have been written into a series of features, one for each library. However, the visual output suggests that it's not happening right so. Perhaps it's because the loop is not iterating through the `library_versions` dictionary separately.
Let's adjust the function to ensure each library and its corresponding version results in a single feature:
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">fme</span>
<span class="kn">import</span> <span class="nn">fmeobjects</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">pkg_resources</span>
<span class="kn">from</span> <span class="nn">distutils.sysconfig</span> <span class="kn">import</span> <span class="n">get_python_lib</span>
<span class="k">class</span> <span class="nc">FeatureProcessor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">has_support_for</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">support_type</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">return</span> <span class="n">support_type</span> <span class="o">==</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FME_SUPPORT_FEATURE_TABLE_SHIM</span>
<span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="c1"># Call get_environment_info instead of self.pyoutput(feature)</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_environment_info</span><span class="p">()</span>
<span class="c1"># Use the dictionary to create a new feature</span>
<span class="c1"># Deal with dictionary within the dictionary separately (i.e., for library_versions)</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">env_info</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="s1">&#39;library_versions&#39;</span><span class="p">:</span>
<span class="k">for</span> <span class="n">lib</span><span class="p">,</span> <span class="n">version</span> <span class="ow">in</span> <span class="n">value</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Library&quot;</span><span class="p">,</span> <span class="n">lib</span><span class="p">)</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Version&quot;</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">process_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Outputs a list of the currently active python environment, location on disk, and version numbers.&quot;&quot;&quot;</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Getting Python Version and location</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="c1"># Getting the local site-packages path</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="c1"># Getting installed libraries</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">working_set</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
In this version, each library is represented by a new feature with two attributes: "Library" (name of the library) and "Version" (version of the library). You should see one feature for each entry in your Python environment — one feature for each library, and one feature each for Python version, location, and site-packages path — all with their corresponding details as attribute values.
If you wish to expose attributes explicitly via "Attributes to Expose" field in the transformer parameters, you should enter `Library, Version, python_version, python_location, site_packages_path`.</p>
<p class="user">now it outputs 230 rows and 5 columns. All cells are empty except for col 'site_packages_path' row 3, which holds "C:\ArcGIS\bin\Python\envs\arcgispro-py3\Lib\site-packages".
The headers are "Library, Version, python_version, python_location, site_packages_path"</p>
<p class="assistant">The issue refers that the attributes of a specific feature are only written when it corresponds to it. This results in a sparse data with lot of None or empty cells.
To make it easier to interpret the result in Visual Preview or any consequent transformations, we should create an individual feature for each piece of information. We can do this by modifying the loop in the `input` method like follows:
<div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="c1"># Get the environment information</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_environment_info</span><span class="p">()</span>
<span class="c1"># Handling general python attributes</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">env_info</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s1">&#39;library_versions&#39;</span><span class="p">:</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Attribute&quot;</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Value&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="c1"># Handling library versions separately</span>
<span class="k">for</span> <span class="n">library</span><span class="p">,</span> <span class="n">version</span> <span class="ow">in</span> <span class="n">env_info</span><span class="p">[</span><span class="s1">&#39;library_versions&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Attribute&quot;</span><span class="p">,</span> <span class="n">library</span><span class="p">)</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Value&quot;</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
</pre></div>
In this updated code, instead of setting `key` as attribute name, it sets "Attribute" as attribute name and `key` as its value. The actual `'Value'` field holds the corresponding value for that attribute or library.
For the "Attributes to Expose" field in the transformer parameters, you should now enter `Attribute, Value`. The output should be now a 2-column dataset with 'Attribute' and 'Value' columns. The attribute names will now be the row values in the `'Attribute'` column, and their associated values will be the row values in the `'Value'` column. This way we achieve a 'long format' data style, which should make the output easier to interpret and organize.
</p>
<p class="user">Ah, success. Now it works.</p>
<p class="assistant">Great to hear that it's working now!
Just for the sake of summarization, what was done here is we gathered various pieces of information about your Python environment – including the Python version, the location of Python executable, the site-packages path, and the version numbers of installed libraries. This information was then written into individual features on an attribute level.
If you have any more questions about FME, PythonCaller or anything else, feel free to ask. Happy data processing with FME!</p>
<style>
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight .hll { background-color: #ffffcc }
.highlight { background: #f8f8f8; }
.highlight .c { color: #408080; font-style: italic } /* Comment */
.highlight .err { border: 1px solid #FF0000 } /* Error */
.highlight .k { color: #008000; font-weight: bold } /* Keyword */
.highlight .o { color: #666666 } /* Operator */
.highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */
.highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */
.highlight .cp { color: #BC7A00 } /* Comment.Preproc */
.highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */
.highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */
.highlight .cs { color: #408080; font-style: italic } /* Comment.Special */
.highlight .gd { color: #A00000 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #FF0000 } /* Generic.Error */
.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */
.highlight .gi { color: #00A000 } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
.highlight .gt { color: #0044DD } /* Generic.Traceback */
.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008000 } /* Keyword.Pseudo */
.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #B00040 } /* Keyword.Type */
.highlight .m { color: #666666 } /* Literal.Number */
.highlight .s { color: #BA2121 } /* Literal.String */
.highlight .na { color: #7D9029 } /* Name.Attribute */
.highlight .nb { color: #008000 } /* Name.Builtin */
.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */
.highlight .no { color: #880000 } /* Name.Constant */
.highlight .nd { color: #AA22FF } /* Name.Decorator */
.highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */
.highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0000FF } /* Name.Function */
.highlight .nl { color: #A0A000 } /* Name.Label */
.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #19177C } /* Name.Variable */
.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #666666 } /* Literal.Number.Bin */
.highlight .mf { color: #666666 } /* Literal.Number.Float */
.highlight .mh { color: #666666 } /* Literal.Number.Hex */
.highlight .mi { color: #666666 } /* Literal.Number.Integer */
.highlight .mo { color: #666666 } /* Literal.Number.Oct */
.highlight .sa { color: #BA2121 } /* Literal.String.Affix */
.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */
.highlight .sc { color: #BA2121 } /* Literal.String.Char */
.highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */
.highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
.highlight .s2 { color: #BA2121 } /* Literal.String.Double */
.highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
.highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */
.highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
.highlight .sx { color: #008000 } /* Literal.String.Other */
.highlight .sr { color: #BB6688 } /* Literal.String.Regex */
.highlight .s1 { color: #BA2121 } /* Literal.String.Single */
.highlight .ss { color: #19177C } /* Literal.String.Symbol */
.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0000FF } /* Name.Function.Magic */
.highlight .vc { color: #19177C } /* Name.Variable.Class */
.highlight .vg { color: #19177C } /* Name.Variable.Global */
.highlight .vi { color: #19177C } /* Name.Variable.Instance */
.highlight .vm { color: #19177C } /* Name.Variable.Magic */
.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
</style>
<p class="user">Below is the default code for PythonCaller. Add a function that outputs a list of the currently active python environment, location on disk, and version numbers of primary modules and libraries important for use in FME.
import fme
import fmeobjects
class FeatureProcessor(object):
"""Template Class Interface:
When using this class, make sure its name is set as the value of the 'Class to Process Features'
transformer parameter.
"""
def __init__(self):
"""Base constructor for class members."""
pass
def has_support_for(self, support_type: int):
"""This method is called by FME to determine if the PythonCaller supports Bulk mode,
which allows for significant performance gains when processing large numbers of features.
Bulk mode cannot always be supported.
More information available in transformer help.
"""
return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM
def input(self, feature: fmeobjects.FMEFeature):
"""This method is called for each feature which enters the PythonCaller.
Processed input features can be emitted from this method using self.pyoutput().
If knowledge of all input features is required for processing, then input features should be
cached to a list instance variable and processed using group processing or in the close() method.
"""
self.pyoutput(feature)
def close(self):
"""This method is called once all the FME Features have been processed from input()."""
pass
def process_group(self):
"""This method is called by FME for each group when group processing mode is enabled.
This implementation should reset any instance variables used for the next group.
Bulk mode should not be enabled when using group processing.
More information available in transformer help.
"""
pass</p>
<p class="assistant">Sure, here's an example of that Python code function within your class. I've used `sys`, `os`, `platform`, `pip` and `importlib` modules to get the required information.
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">fme</span>
<span class="kn">import</span> <span class="nn">fmeobjects</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">pip</span>
<span class="kn">import</span> <span class="nn">importlib</span>
<span class="c1"># Necessary to get the local site-packages path</span>
<span class="kn">from</span> <span class="nn">distutils.sysconfig</span> <span class="kn">import</span> <span class="n">get_python_lib</span>
<span class="k">class</span> <span class="nc">FeatureProcessor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">has_support_for</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">support_type</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">return</span> <span class="n">support_type</span> <span class="o">==</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FME_SUPPORT_FEATURE_TABLE_SHIM</span>
<span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">feature</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">process_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Outputs a list of the currently active python environment, location on disk, and version numbers.&quot;&quot;&quot;</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Getting Python Version and location</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="c1"># Getting the local site-packages path</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="c1"># Getting installed libraries</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pip</span><span class="o">.</span><span class="n">get_installed_distributions</span><span class="p">()</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
Note: This function (`get_environment_info`) would return a dictionary with keys "python_version", "python_location", "site_packages_path", and "library_versions". The "library_versions" is itself a dictionary with keys as the library names and values as their respective versions. You can call this method within `input` method to see the details for each feature.
Please note, this code was written for Python 2 and may need to be adjusted if you are working with Python 3, as some libraries locations are handled differently between versions. Specifically, from Python 3.8 onwards, the `pip.get_installed_distributions()` method has been removed. You could use `pip freeze` command and parse its output, or use a library like `pkg_resources`.</p>
<p class="user">The code runs successfully but FME reports empty output. Here is the relevant log secton:
PythonCreator_Creator (CreationFactory): Created 1 features
Optional `process_group' method not present; not called
PythonCreator_OUTPUT Brancher -1 4 (BranchingFactory): Branched 1 input feature -- 1 feature routed to the target factory, and 0 features routed to the fallback factory.
_CREATOR_BRANCH_TARGET (TeeFactory): Cloned 1 input feature(s) into 1 output feature(s)
Destination Feature Type Routing Correlator (RoutingFactory): Tested 0 input feature(s), wrote 0 output feature(s): 0 matched merge filters, 0 were routed to output, 0 could not be routed.
Final Output Nuker (TeeFactory): Cloned 0 input feature(s) into 0 output feature(s)
</p>
<p class="assistant">It seems like there's been some confusion. The function `get_environment_info` is merely returning the dictionary of the information about the environment, but it isn't outputting this data as a feature nor writing it into the logs.
To output this data as a feature, you would need to call `get_environment_info` within `input` or `close` methods and utilize `fmeobjects.FMEFeature` to transform this data into a format FME can handle.
Here's a possible way to do this with making use of your `input` method:
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">fme</span>
<span class="kn">import</span> <span class="nn">fmeobjects</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">pip</span>
<span class="kn">import</span> <span class="nn">importlib</span>
<span class="kn">from</span> <span class="nn">distutils.sysconfig</span> <span class="kn">import</span> <span class="n">get_python_lib</span>
<span class="k">class</span> <span class="nc">FeatureProcessor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">has_support_for</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">support_type</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">return</span> <span class="n">support_type</span> <span class="o">==</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FME_SUPPORT_FEATURE_TABLE_SHIM</span>
<span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="c1"># Call get_environment_info instead of self.pyoutput(feature)</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_environment_info</span><span class="p">()</span>
<span class="c1"># Use the dictionary to create a new feature</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">env_info</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">process_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pip</span><span class="o">.</span><span class="n">get_installed_distributions</span><span class="p">()</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
This will output each piece of information about the environment as a separate feature. In the `input` method, I've converted the `env_info` dictionary to a series of FME features which are output via `self.pyoutput`. Each resulting feature has one attribute, with the key and the associated value.
Note that values that are dictionaries or lists need to be converted to string via `str` to be compatible with `FMEFeature`. Hence, the library versions will appear as a string representation of a dictionary.
With these changes, you should be able to start seeing output features in your workspace.</p>
<p class="user">Python Exception <AttributeError>: module 'pip' has no attribute 'get_installed_distributions'
</p>
<p class="assistant">The method `pip.get_installed_distributions()` was removed starting with pip version 10.0. Instead, you could use `pkg_resources` to get a list of installed distributions. Here's your updated `get_environment_info` method:
<div class="highlight"><pre><span></span> <span class="kn">import</span> <span class="nn">pkg_resources</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Outputs a list of the currently active python environment, location on disk, and version numbers.&quot;&quot;&quot;</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Getting Python Version and location</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="c1"># Getting the local site-packages path</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="c1"># Getting installed libraries</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">working_set</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
This will get you the same info as the previous version. Note that `pkg_resources.working_set` is a list of all installed distributions.
</p>
<p class="user">what needs to be entered in to the "Attributes to Expose" field of the "Output Attributes" section of the PythonCaller dialog to use the values returned by the code?</p>
<p class="assistant">The "Attributes to Expose" field specifies which Python variables you want to expose as attributes in the FME feature schema. These exposed attributes will be viewable and usable within your FME workspace.
Since you are outputting attributes dynamically by setting them directly on the FMEFeature object within the `input` method, you do not need to specify anything in the 'Attributes to Expose' field. The created attributes ("python_version", "python_location", "site_packages_path" and "library_versions") will be added to the feature regardless of what is specified in the 'Attributes to Expose' field.
If you were returning a dictionary from the `input` method, you would need to specify the keys of the dictionary in the 'Attributes to Expose' field. But since you are creating attributes directly on the features by using `setAttribute`, this is not necessary.
However, if you still want to explicity expose attribute via "Attributes to Expose" field, you could enter a comma-separated list like so:
```
python_version, python_location, site_packages_path, library_versions
```
Ensure these align exactly with the keys in the dictionary returned by `get_environment_info`, including case sensitivity.</p>
<p class="user">When I leave Attributes to Expose empty I get output of 4 rows and 1 column, all values empty.
When I enter `python_version, python_location, site_packages_path, library_versions` I get the output in Visual Preview. (A 4 by 4 table matrix, white space delimited, all cells empty except the bottom right, corresponding to col `library_versions` and row 3):
```
python_version, python_location, site_packages_path, library_versions
{'fme-emailer': '3.3.3', 'certifi': '2022.9.24', 'regex': '2022.7.9', 'fsspec': '2022.7.1', 'dask': '2022.7.0', 'distributed': '2022.7.0', 'pefile': '2022.5.30', 'setuptools': '65.5.1', 'cryptography': '38.0.4', 'pyzmq': '24.0.1', 'keyring': '23.4.0', 'pip': '22.2.2', 'pyopenssl': '22.0.0', 'attrs': '21.4.0', 'argon2-cffi': '21.3.0', 'packaging': '21.3', 'argon2-cffi-bindings': '21.2.0', 'azure-storage-blob': '12.8.0', 'pillow': '9.5.0', 'click': '8.0.4', 'ipython': '7.34.0', 'ipywidgets': '7.7.2', 'nbconvert': '7.2.1', 'pytest': '7.2.0', 'ipykernel': '6.16.0', 'notebook': '6.4.12', 'jupyter-console': '6.4.2', 'jupyter-client': '6.1.12', 'tornado': '6.1', 'pyyaml': '6.0', 'psutil': '5.9.0', 'nbformat': '5.6.1', 'traitlets': '5.5.0', 'ujson': '5.4.0', 'decorator': '5.1.1', 'tqdm': '4.64.1', 'fonttools': '4.25.0', 'jsonschema': '4.16.0', 'importlib-metadata': '4.11.3', 'jupyter-core': '4.11.2', 'beautifulsoup4': '4.11.1', 'lxml': '4.9.1', 'saspy': '4.3.2', 'typing-extensions': '4.3.0', 'cachetools': '4.2.2', 'bleach': '4.1.0', 'pyodbc': '4.0.34', 'flake8': '4.0.1', 'chardet': '4.0.0', 'lerc': '4.0', 'protobuf': '3.19.4', 'zipp': '3.8.0', 'h5py': '3.7.0', 'widgetsnbextension': '3.6.1', 'matplotlib': '3.6.0', 'anyio': '3.5.0', 'jupyterlab': '3.4.8', 'idna': '3.4', 'oauthlib': '3.2.0', 'lz4': '3.1.3', 'pro-notebook-integration': '3.1', 'openpyxl': '3.0.10', 'pyparsing': '3.0.9', 'prompt-toolkit': '3.0.5', 'jinja2': '3.0.3', 'jupyterlab-widgets': '3.0.3', 'requests': '2.28.1', 'pycparser': '2.21', 'fastjsonschema': '2.16.2', 'jupyterlab-server': '2.16.0', 'pygments': '2.9.0', 'networkx': '2.8.4', 'python-dateutil': '2.8.2', 'numexpr': '2.8.1', 'pycodestyle': '2.8.0', 'platformdirs': '2.5.2', 'affine': '2.4.0', 'pyflakes': '2.4.0', 'pyjwt': '2.4.0', 'sortedcontainers': '2.4.0', 'soupsieve': '2.3.2.post1', 'mkl-service': '2.3.0', 'pyshp': '2.1.3', 'markupsafe': '2.1.1', 'arcgis': '2.1.0.2', 'zict': '2.1.0', 'asttokens': '2.0.5', 'charset-normalizer': '2.0.4', 'mistune': '2.0.4', 'pywinpty': '2.0.1', 'shapely': '2.0.1', 'tomli': '2.0.1', 'xlrd': '2.0.1', 'cloudpickle': '2.0.0', 'botocore': '1.29.76', 'boto3': '1.26.76', 'urllib3': '1.26.12', 'pypdf2': '1.26.0', 'jupyter-server': '1.21.0', 'numpy': '1.20.1', 'six': '1.16.0', 'cffi': '1.15.1', 'wrapt': '1.14.1', 'azure-core': '1.12.0', 'swat': '1.12.0', 'gssapi': '1.8.1', 'send2trash': '1.8.0', 'pysocks': '1.7.1', 'tblib': '1.7.0', 'python-certifi-win32': '1.7.dev1+g24a5d1b', 'cftime': '1.6.2', 'scipy': '1.6.2', 'netcdf4': '1.6.1', 'pystac': '1.6.1', 'nest-asyncio': '1.5.5', 'debugpy': '1.5.1', 'pandocfilters': '1.5.0', 'sqlalchemy': '1.4.39', 'snuggs': '1.4.7', 'jupyter-latex-envs': '1.4.6', 'appdirs': '1.4.4', 'pandas': '1.4.4', 'typed-ast': '1.4.3', 'kiwisolver': '1.4.2', 'jdcal': '1.4.1', 'blinker': '1.4', 'ntlm-auth': '1.4.0', 'nose': '1.3.7', 'rasterio': '1.3.6', 'bottleneck': '1.3.5', 'mkl-fft': '1.3.0', 'xlwt': '1.3.0', 'requests-oauthlib': '1.3.0', 'requests-gssapi': '1.2.3', 'mkl-random': '1.2.2', 'mpmath': '1.2.1', 'partd': '1.2.0', 'sniffio': '1.2.0', 'munkres': '1.1.4', 'click-plugins': '1.1.1', 'greenlet': '1.1.1', 'iniconfig': '1.1.1', 'tinycss2': '1.1.1', 'et-xmlfile': '1.1.0', 'win-inet-pton': '1.1.0', 'contourpy': '1.0.5', 'msgpack': '1.0.3', 'heapdict': '1.0.1', 'jmespath': '1.0.1', 'geomet': '1.0.0', 'locket': '1.0.0', 'exceptiongroup': '1.0.0rc9', 'websocket-client': '0.58.0', 'olefile': '0.46', 'wheel': '0.37.1', 'xarray': '0.20.1', 'pyrsistent': '0.18.0', 'pluggy': '0.13.1', 'terminado': '0.13.1', 'seaborn': '0.12.1', 'requests-kerberos': '0.12.0', 'toolz': '0.11.2', 'cytoolz': '0.11.0', 'cycler': '0.11.0', 'toml': '0.10.2', 'json5': '0.9.5', 'requests-toolbelt': '0.9.1', 'pathspec': '0.9.0', 'executing': '0.8.3', 'parso': '0.8.3', 'prometheus-client': '0.8.0', 'winkerberos': '0.8.0', 'pickleshare': '0.7.5', 'cligj': '0.7.2', 'defusedxml': '0.7.1', 'brotlipy': '0.7.0', 'mccabe': '0.7.0', 'nbclient': '0.7.0', 'msrest': '0.6.21', 'isodate': '0.6.0', 'jupyter-contrib-nbextensions': '0.6.0', 's3transfer': '0.6.0', 'webencodings': '0.5.1', 'stack-data': '0.5.1', 'testpath': '0.5.0', 'nbclassic': '0.4.8', 'colorama': '0.4.5', 'mypy-extensions': '0.4.3', 'jupyter-nbextensions-configurator': '0.4.1', 'x86cpu': '0.4', 'entrypoints': '0.4', 'jupyter-contrib-core': '0.4.0', 'wcwidth': '0.2.5', 'jupyterlab-pygments': '0.2.2', 'pure-eval': '0.2.2', 'pywin32-ctypes': '0.2.0', 'wincertstore': '0.2', 'backcall': '0.2.0', 'despatch': '0.2.0', 'ipython-genutils': '0.2.0', 'jupyter-highlight-selected-word': '0.2.0', 'matplotlib-inline': '0.1.6', 'notebook-shim': '0.1.0', 'black': '0.0.0', 'requests-negotiate-sspi': '0.0.0', 'babel': '2.11.0', 'future': '0.18.2', 'jedi': '0.18.2', 'pytz': '2022.6', 'sympy': '1.9', 'filelock': '3.10.6', 'tldextract': '3.4.0', 'markdown': '3.1.1', 'xlsxwriter': '3.0.9', 'tzlocal': '2.1', 'hdfs': '2.0.16', 'diskcache': '2.0.2', 'requests-file': '1.5.1', 'requests-aws4auth': '1.2.2', 'requests-ntlm': '1.2.0', 'pypac': '0.16.1', 'xmltodict': '0.13.0', 'python-docx': '0.8.11', 'pyspnego': '0.8.0', 'python-pptx': '0.6.21', 'docopt': '0.6.2', 'dukpy': '0.3.0'}
```
</p>
<p class="assistant">This error suggests that the loop handling creating new features and setting attributes is not working correctly. The result "`library_versions`" should have been written into a series of features, one for each library. However, the visual output suggests that it's not happening right so. Perhaps it's because the loop is not iterating through the `library_versions` dictionary separately.
Let's adjust the function to ensure each library and its corresponding version results in a single feature:
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">fme</span>
<span class="kn">import</span> <span class="nn">fmeobjects</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">pkg_resources</span>
<span class="kn">from</span> <span class="nn">distutils.sysconfig</span> <span class="kn">import</span> <span class="n">get_python_lib</span>
<span class="k">class</span> <span class="nc">FeatureProcessor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">has_support_for</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">support_type</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">return</span> <span class="n">support_type</span> <span class="o">==</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FME_SUPPORT_FEATURE_TABLE_SHIM</span>
<span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="c1"># Call get_environment_info instead of self.pyoutput(feature)</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_environment_info</span><span class="p">()</span>
<span class="c1"># Use the dictionary to create a new feature</span>
<span class="c1"># Deal with dictionary within the dictionary separately (i.e., for library_versions)</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">env_info</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="s1">&#39;library_versions&#39;</span><span class="p">:</span>
<span class="k">for</span> <span class="n">lib</span><span class="p">,</span> <span class="n">version</span> <span class="ow">in</span> <span class="n">value</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Library&quot;</span><span class="p">,</span> <span class="n">lib</span><span class="p">)</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Version&quot;</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">process_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span> <span class="nf">get_environment_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Outputs a list of the currently active python environment, location on disk, and version numbers.&quot;&quot;&quot;</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Getting Python Version and location</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;python_location&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
<span class="c1"># Getting the local site-packages path</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;site_packages_path&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_python_lib</span><span class="p">()</span>
<span class="c1"># Getting installed libraries</span>
<span class="n">installed_libraries</span> <span class="o">=</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">working_set</span>
<span class="n">library_versions</span> <span class="o">=</span> <span class="p">{</span><span class="n">library</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">library</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">library</span> <span class="ow">in</span> <span class="n">installed_libraries</span><span class="p">}</span>
<span class="n">env_info</span><span class="p">[</span><span class="s2">&quot;library_versions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">library_versions</span>
<span class="k">return</span> <span class="n">env_info</span>
</pre></div>
In this version, each library is represented by a new feature with two attributes: "Library" (name of the library) and "Version" (version of the library). You should see one feature for each entry in your Python environment — one feature for each library, and one feature each for Python version, location, and site-packages path — all with their corresponding details as attribute values.
If you wish to expose attributes explicitly via "Attributes to Expose" field in the transformer parameters, you should enter `Library, Version, python_version, python_location, site_packages_path`.</p>
<p class="user">now it outputs 230 rows and 5 columns. All cells are empty except for col 'site_packages_path' row 3, which holds "C:\ArcGIS\bin\Python\envs\arcgispro-py3\Lib\site-packages".
The headers are "Library, Version, python_version, python_location, site_packages_path"</p>
<p class="assistant">The issue refers that the attributes of a specific feature are only written when it corresponds to it. This results in a sparse data with lot of None or empty cells.
To make it easier to interpret the result in Visual Preview or any consequent transformations, we should create an individual feature for each piece of information. We can do this by modifying the loop in the `input` method like follows:
<div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feature</span><span class="p">:</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">):</span>
<span class="c1"># Get the environment information</span>
<span class="n">env_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_environment_info</span><span class="p">()</span>
<span class="c1"># Handling general python attributes</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">env_info</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s1">&#39;library_versions&#39;</span><span class="p">:</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Attribute&quot;</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Value&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
<span class="c1"># Handling library versions separately</span>
<span class="k">for</span> <span class="n">library</span><span class="p">,</span> <span class="n">version</span> <span class="ow">in</span> <span class="n">env_info</span><span class="p">[</span><span class="s1">&#39;library_versions&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">new_feature</span> <span class="o">=</span> <span class="n">fmeobjects</span><span class="o">.</span><span class="n">FMEFeature</span><span class="p">()</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Attribute&quot;</span><span class="p">,</span> <span class="n">library</span><span class="p">)</span>
<span class="n">new_feature</span><span class="o">.</span><span class="n">setAttribute</span><span class="p">(</span><span class="s2">&quot;Value&quot;</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pyoutput</span><span class="p">(</span><span class="n">new_feature</span><span class="p">)</span>
</pre></div>
In this updated code, instead of setting `key` as attribute name, it sets "Attribute" as attribute name and `key` as its value. The actual `'Value'` field holds the corresponding value for that attribute or library.
For the "Attributes to Expose" field in the transformer parameters, you should now enter `Attribute, Value`. The output should be now a 2-column dataset with 'Attribute' and 'Value' columns. The attribute names will now be the row values in the `'Attribute'` column, and their associated values will be the row values in the `'Value'` column. This way we achieve a 'long format' data style, which should make the output easier to interpret and organize.
</p>
<p class="user">Ah, success. Now it works.</p>
<p class="assistant">Great to hear that it's working now!
Just for the sake of summarization, what was done here is we gathered various pieces of information about your Python environment – including the Python version, the location of Python executable, the site-packages path, and the version numbers of installed libraries. This information was then written into individual features on an attribute level.
If you have any more questions about FME, PythonCaller or anything else, feel free to ask. Happy data processing with FME!</p>
</body>
</html>
Raw
session.json
{
"folderId": null,
"id": "f3b60060-cab1-4135-b007-6c7494be68e5",
"messages": [
{
"content": "Below is the default code for PythonCaller. Add a function that outputs a list of the currently active python environment, location on disk, and version numbers of primary modules and libraries important for use in FME.\n\nimport fme\nimport fmeobjects\n\n\nclass FeatureProcessor(object):\n \"\"\"Template Class Interface:\n When using this class, make sure its name is set as the value of the 'Class to Process Features'\n transformer parameter.\n \"\"\"\n\n def __init__(self):\n \"\"\"Base constructor for class members.\"\"\"\n pass\n \n def has_support_for(self, support_type: int):\n \"\"\"This method is called by FME to determine if the PythonCaller supports Bulk mode,\n which allows for significant performance gains when processing large numbers of features.\n Bulk mode cannot always be supported. \n More information available in transformer help.\n \"\"\"\n return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM\n \n def input(self, feature: fmeobjects.FMEFeature):\n \"\"\"This method is called for each feature which enters the PythonCaller. \n Processed input features can be emitted from this method using self.pyoutput().\n If knowledge of all input features is required for processing, then input features should be \n cached to a list instance variable and processed using group processing or in the close() method.\n \"\"\"\n self.pyoutput(feature)\n\n def close(self):\n \"\"\"This method is called once all the FME Features have been processed from input().\"\"\"\n pass\n\n def process_group(self):\n \"\"\"This method is called by FME for each group when group processing mode is enabled.\n This implementation should reset any instance variables used for the next group. \n Bulk mode should not be enabled when using group processing. \n More information available in transformer help.\n \"\"\"\n pass",
"role": "user"
},
{
"content": "Sure, here's an example of that Python code function within your class. I've used `sys`, `os`, `platform`, `pip` and `importlib` modules to get the required information.\n\n```python\nimport fme\nimport fmeobjects\nimport sys\nimport os\nimport platform\nimport pip\nimport importlib\n\n# Necessary to get the local site-packages path\nfrom distutils.sysconfig import get_python_lib\n\nclass FeatureProcessor(object):\n\n def __init__(self):\n pass\n \n def has_support_for(self, support_type: int):\n return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM\n \n def input(self, feature: fmeobjects.FMEFeature):\n self.pyoutput(feature)\n \n def close(self):\n pass\n \n def process_group(self):\n pass\n\n def get_environment_info(self):\n \"\"\"Outputs a list of the currently active python environment, location on disk, and version numbers.\"\"\"\n env_info = {}\n \n # Getting Python Version and location\n env_info[\"python_version\"] = platform.python_version()\n env_info[\"python_location\"] = sys.executable\n \n # Getting the local site-packages path\n env_info[\"site_packages_path\"] = get_python_lib()\n \n # Getting installed libraries\n installed_libraries = pip.get_installed_distributions()\n library_versions = {library.key: library.version for library in installed_libraries}\n env_info[\"library_versions\"] = library_versions\n \n return env_info\n```\n\nNote: This function (`get_environment_info`) would return a dictionary with keys \"python_version\", \"python_location\", \"site_packages_path\", and \"library_versions\". The \"library_versions\" is itself a dictionary with keys as the library names and values as their respective versions. You can call this method within `input` method to see the details for each feature. \n\nPlease note, this code was written for Python 2 and may need to be adjusted if you are working with Python 3, as some libraries locations are handled differently between versions. Specifically, from Python 3.8 onwards, the `pip.get_installed_distributions()` method has been removed. You could use `pip freeze` command and parse its output, or use a library like `pkg_resources`.",
"role": "assistant"
},
{
"content": "The code runs successfully but FME reports empty output. Here is the relevant log secton:\n\nPythonCreator_Creator (CreationFactory): Created 1 features\nOptional `process_group' method not present; not called\nPythonCreator_OUTPUT Brancher -1 4 (BranchingFactory): Branched 1 input feature -- 1 feature routed to the target factory, and 0 features routed to the fallback factory.\n_CREATOR_BRANCH_TARGET (TeeFactory): Cloned 1 input feature(s) into 1 output feature(s)\nDestination Feature Type Routing Correlator (RoutingFactory): Tested 0 input feature(s), wrote 0 output feature(s): 0 matched merge filters, 0 were routed to output, 0 could not be routed.\nFinal Output Nuker (TeeFactory): Cloned 0 input feature(s) into 0 output feature(s)\n",
"role": "user"
},
{
"content": "It seems like there's been some confusion. The function `get_environment_info` is merely returning the dictionary of the information about the environment, but it isn't outputting this data as a feature nor writing it into the logs.\n\nTo output this data as a feature, you would need to call `get_environment_info` within `input` or `close` methods and utilize `fmeobjects.FMEFeature` to transform this data into a format FME can handle.\n\nHere's a possible way to do this with making use of your `input` method:\n\n```python\nimport fme\nimport fmeobjects\nimport sys\nimport os\nimport platform\nimport pip\nimport importlib\nfrom distutils.sysconfig import get_python_lib\n\nclass FeatureProcessor(object):\n\n def __init__(self):\n pass\n \n def has_support_for(self, support_type: int):\n return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM\n\n def input(self, feature: fmeobjects.FMEFeature):\n # Call get_environment_info instead of self.pyoutput(feature)\n env_info = self.get_environment_info()\n\n # Use the dictionary to create a new feature\n for key, value in env_info.items():\n new_feature = fmeobjects.FMEFeature()\n new_feature.setAttribute(key, str(value))\n self.pyoutput(new_feature)\n\n def close(self):\n pass\n\n def process_group(self):\n pass\n \n def get_environment_info(self):\n env_info = {}\n env_info[\"python_version\"] = platform.python_version()\n env_info[\"python_location\"] = sys.executable\n env_info[\"site_packages_path\"] = get_python_lib()\n installed_libraries = pip.get_installed_distributions()\n library_versions = {library.key: library.version for library in installed_libraries}\n env_info[\"library_versions\"] = library_versions\n\n return env_info\n```\n\nThis will output each piece of information about the environment as a separate feature. In the `input` method, I've converted the `env_info` dictionary to a series of FME features which are output via `self.pyoutput`. Each resulting feature has one attribute, with the key and the associated value.\n\nNote that values that are dictionaries or lists need to be converted to string via `str` to be compatible with `FMEFeature`. Hence, the library versions will appear as a string representation of a dictionary.\n\nWith these changes, you should be able to start seeing output features in your workspace.",
"role": "assistant"
},
{
"content": "Python Exception <AttributeError>: module 'pip' has no attribute 'get_installed_distributions'\n",
"role": "user"
},
{
"content": "The method `pip.get_installed_distributions()` was removed starting with pip version 10.0. Instead, you could use `pkg_resources` to get a list of installed distributions. Here's your updated `get_environment_info` method:\n\n```python\n import pkg_resources\n \n def get_environment_info(self):\n \"\"\"Outputs a list of the currently active python environment, location on disk, and version numbers.\"\"\"\n env_info = {}\n\n # Getting Python Version and location\n env_info[\"python_version\"] = platform.python_version()\n env_info[\"python_location\"] = sys.executable\n\n # Getting the local site-packages path\n env_info[\"site_packages_path\"] = get_python_lib()\n\n # Getting installed libraries\n installed_libraries = pkg_resources.working_set\n library_versions = {library.key: library.version for library in installed_libraries}\n env_info[\"library_versions\"] = library_versions\n\n return env_info\n``` \n\nThis will get you the same info as the previous version. Note that `pkg_resources.working_set` is a list of all installed distributions.\n",
"role": "assistant"
},
{
"content": "what needs to be entered in to the \"Attributes to Expose\" field of the \"Output Attributes\" section of the PythonCaller dialog to use the values returned by the code?",
"role": "user"
},
{
"content": "The \"Attributes to Expose\" field specifies which Python variables you want to expose as attributes in the FME feature schema. These exposed attributes will be viewable and usable within your FME workspace.\n\nSince you are outputting attributes dynamically by setting them directly on the FMEFeature object within the `input` method, you do not need to specify anything in the 'Attributes to Expose' field. The created attributes (\"python_version\", \"python_location\", \"site_packages_path\" and \"library_versions\") will be added to the feature regardless of what is specified in the 'Attributes to Expose' field.\n\nIf you were returning a dictionary from the `input` method, you would need to specify the keys of the dictionary in the 'Attributes to Expose' field. But since you are creating attributes directly on the features by using `setAttribute`, this is not necessary.\n\nHowever, if you still want to explicity expose attribute via \"Attributes to Expose\" field, you could enter a comma-separated list like so:\n```\npython_version, python_location, site_packages_path, library_versions\n```\n\nEnsure these align exactly with the keys in the dictionary returned by `get_environment_info`, including case sensitivity.",
"role": "assistant"
},
{
"content": "When I leave Attributes to Expose empty I get output of 4 rows and 1 column, all values empty.\nWhen I enter `python_version, python_location, site_packages_path, library_versions` I get the output in Visual Preview. (A 4 by 4 table matrix, white space delimited, all cells empty except the bottom right, corresponding to col `library_versions` and row 3):\n\n```\npython_version,\tpython_location,\tsite_packages_path,\tlibrary_versions\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t{'fme-emailer': '3.3.3', 'certifi': '2022.9.24', 'regex': '2022.7.9', 'fsspec': '2022.7.1', 'dask': '2022.7.0', 'distributed': '2022.7.0', 'pefile': '2022.5.30', 'setuptools': '65.5.1', 'cryptography': '38.0.4', 'pyzmq': '24.0.1', 'keyring': '23.4.0', 'pip': '22.2.2', 'pyopenssl': '22.0.0', 'attrs': '21.4.0', 'argon2-cffi': '21.3.0', 'packaging': '21.3', 'argon2-cffi-bindings': '21.2.0', 'azure-storage-blob': '12.8.0', 'pillow': '9.5.0', 'click': '8.0.4', 'ipython': '7.34.0', 'ipywidgets': '7.7.2', 'nbconvert': '7.2.1', 'pytest': '7.2.0', 'ipykernel': '6.16.0', 'notebook': '6.4.12', 'jupyter-console': '6.4.2', 'jupyter-client': '6.1.12', 'tornado': '6.1', 'pyyaml': '6.0', 'psutil': '5.9.0', 'nbformat': '5.6.1', 'traitlets': '5.5.0', 'ujson': '5.4.0', 'decorator': '5.1.1', 'tqdm': '4.64.1', 'fonttools': '4.25.0', 'jsonschema': '4.16.0', 'importlib-metadata': '4.11.3', 'jupyter-core': '4.11.2', 'beautifulsoup4': '4.11.1', 'lxml': '4.9.1', 'saspy': '4.3.2', 'typing-extensions': '4.3.0', 'cachetools': '4.2.2', 'bleach': '4.1.0', 'pyodbc': '4.0.34', 'flake8': '4.0.1', 'chardet': '4.0.0', 'lerc': '4.0', 'protobuf': '3.19.4', 'zipp': '3.8.0', 'h5py': '3.7.0', 'widgetsnbextension': '3.6.1', 'matplotlib': '3.6.0', 'anyio': '3.5.0', 'jupyterlab': '3.4.8', 'idna': '3.4', 'oauthlib': '3.2.0', 'lz4': '3.1.3', 'pro-notebook-integration': '3.1', 'openpyxl': '3.0.10', 'pyparsing': '3.0.9', 'prompt-toolkit': '3.0.5', 'jinja2': '3.0.3', 'jupyterlab-widgets': '3.0.3', 'requests': '2.28.1', 'pycparser': '2.21', 'fastjsonschema': '2.16.2', 'jupyterlab-server': '2.16.0', 'pygments': '2.9.0', 'networkx': '2.8.4', 'python-dateutil': '2.8.2', 'numexpr': '2.8.1', 'pycodestyle': '2.8.0', 'platformdirs': '2.5.2', 'affine': '2.4.0', 'pyflakes': '2.4.0', 'pyjwt': '2.4.0', 'sortedcontainers': '2.4.0', 'soupsieve': '2.3.2.post1', 'mkl-service': '2.3.0', 'pyshp': '2.1.3', 'markupsafe': '2.1.1', 'arcgis': '2.1.0.2', 'zict': '2.1.0', 'asttokens': '2.0.5', 'charset-normalizer': '2.0.4', 'mistune': '2.0.4', 'pywinpty': '2.0.1', 'shapely': '2.0.1', 'tomli': '2.0.1', 'xlrd': '2.0.1', 'cloudpickle': '2.0.0', 'botocore': '1.29.76', 'boto3': '1.26.76', 'urllib3': '1.26.12', 'pypdf2': '1.26.0', 'jupyter-server': '1.21.0', 'numpy': '1.20.1', 'six': '1.16.0', 'cffi': '1.15.1', 'wrapt': '1.14.1', 'azure-core': '1.12.0', 'swat': '1.12.0', 'gssapi': '1.8.1', 'send2trash': '1.8.0', 'pysocks': '1.7.1', 'tblib': '1.7.0', 'python-certifi-win32': '1.7.dev1+g24a5d1b', 'cftime': '1.6.2', 'scipy': '1.6.2', 'netcdf4': '1.6.1', 'pystac': '1.6.1', 'nest-asyncio': '1.5.5', 'debugpy': '1.5.1', 'pandocfilters': '1.5.0', 'sqlalchemy': '1.4.39', 'snuggs': '1.4.7', 'jupyter-latex-envs': '1.4.6', 'appdirs': '1.4.4', 'pandas': '1.4.4', 'typed-ast': '1.4.3', 'kiwisolver': '1.4.2', 'jdcal': '1.4.1', 'blinker': '1.4', 'ntlm-auth': '1.4.0', 'nose': '1.3.7', 'rasterio': '1.3.6', 'bottleneck': '1.3.5', 'mkl-fft': '1.3.0', 'xlwt': '1.3.0', 'requests-oauthlib': '1.3.0', 'requests-gssapi': '1.2.3', 'mkl-random': '1.2.2', 'mpmath': '1.2.1', 'partd': '1.2.0', 'sniffio': '1.2.0', 'munkres': '1.1.4', 'click-plugins': '1.1.1', 'greenlet': '1.1.1', 'iniconfig': '1.1.1', 'tinycss2': '1.1.1', 'et-xmlfile': '1.1.0', 'win-inet-pton': '1.1.0', 'contourpy': '1.0.5', 'msgpack': '1.0.3', 'heapdict': '1.0.1', 'jmespath': '1.0.1', 'geomet': '1.0.0', 'locket': '1.0.0', 'exceptiongroup': '1.0.0rc9', 'websocket-client': '0.58.0', 'olefile': '0.46', 'wheel': '0.37.1', 'xarray': '0.20.1', 'pyrsistent': '0.18.0', 'pluggy': '0.13.1', 'terminado': '0.13.1', 'seaborn': '0.12.1', 'requests-kerberos': '0.12.0', 'toolz': '0.11.2', 'cytoolz': '0.11.0', 'cycler': '0.11.0', 'toml': '0.10.2', 'json5': '0.9.5', 'requests-toolbelt': '0.9.1', 'pathspec': '0.9.0', 'executing': '0.8.3', 'parso': '0.8.3', 'prometheus-client': '0.8.0', 'winkerberos': '0.8.0', 'pickleshare': '0.7.5', 'cligj': '0.7.2', 'defusedxml': '0.7.1', 'brotlipy': '0.7.0', 'mccabe': '0.7.0', 'nbclient': '0.7.0', 'msrest': '0.6.21', 'isodate': '0.6.0', 'jupyter-contrib-nbextensions': '0.6.0', 's3transfer': '0.6.0', 'webencodings': '0.5.1', 'stack-data': '0.5.1', 'testpath': '0.5.0', 'nbclassic': '0.4.8', 'colorama': '0.4.5', 'mypy-extensions': '0.4.3', 'jupyter-nbextensions-configurator': '0.4.1', 'x86cpu': '0.4', 'entrypoints': '0.4', 'jupyter-contrib-core': '0.4.0', 'wcwidth': '0.2.5', 'jupyterlab-pygments': '0.2.2', 'pure-eval': '0.2.2', 'pywin32-ctypes': '0.2.0', 'wincertstore': '0.2', 'backcall': '0.2.0', 'despatch': '0.2.0', 'ipython-genutils': '0.2.0', 'jupyter-highlight-selected-word': '0.2.0', 'matplotlib-inline': '0.1.6', 'notebook-shim': '0.1.0', 'black': '0.0.0', 'requests-negotiate-sspi': '0.0.0', 'babel': '2.11.0', 'future': '0.18.2', 'jedi': '0.18.2', 'pytz': '2022.6', 'sympy': '1.9', 'filelock': '3.10.6', 'tldextract': '3.4.0', 'markdown': '3.1.1', 'xlsxwriter': '3.0.9', 'tzlocal': '2.1', 'hdfs': '2.0.16', 'diskcache': '2.0.2', 'requests-file': '1.5.1', 'requests-aws4auth': '1.2.2', 'requests-ntlm': '1.2.0', 'pypac': '0.16.1', 'xmltodict': '0.13.0', 'python-docx': '0.8.11', 'pyspnego': '0.8.0', 'python-pptx': '0.6.21', 'docopt': '0.6.2', 'dukpy': '0.3.0'}\n```\n",
"role": "user"
},
{
"content": "This error suggests that the loop handling creating new features and setting attributes is not working correctly. The result \"`library_versions`\" should have been written into a series of features, one for each library. However, the visual output suggests that it's not happening right so. Perhaps it's because the loop is not iterating through the `library_versions` dictionary separately. \n\nLet's adjust the function to ensure each library and its corresponding version results in a single feature:\n\n```python\nimport fme\nimport fmeobjects\nimport sys\nimport os\nimport platform\nimport pkg_resources\nfrom distutils.sysconfig import get_python_lib\n\nclass FeatureProcessor(object):\n\n def __init__(self):\n pass\n\n def has_support_for(self, support_type: int):\n return support_type == fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM\n\n def input(self, feature: fmeobjects.FMEFeature):\n # Call get_environment_info instead of self.pyoutput(feature)\n env_info = self.get_environment_info()\n\n # Use the dictionary to create a new feature\n # Deal with dictionary within the dictionary separately (i.e., for library_versions)\n for key, value in env_info.items():\n if key == 'library_versions':\n for lib, version in value.items():\n new_feature = fmeobjects.FMEFeature()\n new_feature.setAttribute(\"Library\", lib)\n new_feature.setAttribute(\"Version\", version)\n self.pyoutput(new_feature)\n else:\n new_feature = fmeobjects.FMEFeature()\n new_feature.setAttribute(key, str(value))\n self.pyoutput(new_feature)\n\n def close(self):\n pass\n\n def process_group(self):\n pass\n\n def get_environment_info(self):\n \"\"\"Outputs a list of the currently active python environment, location on disk, and version numbers.\"\"\"\n env_info = {}\n\n # Getting Python Version and location\n env_info[\"python_version\"] = platform.python_version()\n env_info[\"python_location\"] = sys.executable\n\n # Getting the local site-packages path\n env_info[\"site_packages_path\"] = get_python_lib()\n\n # Getting installed libraries\n installed_libraries = pkg_resources.working_set\n library_versions = {library.key: library.version for library in installed_libraries}\n env_info[\"library_versions\"] = library_versions\n\n return env_info\n```\n\nIn this version, each library is represented by a new feature with two attributes: \"Library\" (name of the library) and \"Version\" (version of the library). You should see one feature for each entry in your Python environment \u2014 one feature for each library, and one feature each for Python version, location, and site-packages path \u2014 all with their corresponding details as attribute values. \n\nIf you wish to expose attributes explicitly via \"Attributes to Expose\" field in the transformer parameters, you should enter `Library, Version, python_version, python_location, site_packages_path`.",
"role": "assistant"
},
{
"content": "now it outputs 230 rows and 5 columns. All cells are empty except for col 'site_packages_path' row 3, which holds \"C:\\ArcGIS\\bin\\Python\\envs\\arcgispro-py3\\Lib\\site-packages\". \nThe headers are \"Library,\tVersion,\tpython_version,\tpython_location,\tsite_packages_path\"",
"role": "user"
},
{
"content": "The issue refers that the attributes of a specific feature are only written when it corresponds to it. This results in a sparse data with lot of None or empty cells. \nTo make it easier to interpret the result in Visual Preview or any consequent transformations, we should create an individual feature for each piece of information. We can do this by modifying the loop in the `input` method like follows:\n\n```python\ndef input(self, feature: fmeobjects.FMEFeature):\n # Get the environment information\n env_info = self.get_environment_info()\n\n # Handling general python attributes\n for key, value in env_info.items():\n if key != 'library_versions':\n new_feature = fmeobjects.FMEFeature()\n new_feature.setAttribute(\"Attribute\", key)\n new_feature.setAttribute(\"Value\", str(value))\n self.pyoutput(new_feature)\n\n # Handling library versions separately\n for library, version in env_info['library_versions'].items():\n new_feature = fmeobjects.FMEFeature()\n new_feature.setAttribute(\"Attribute\", library)\n new_feature.setAttribute(\"Value\", version)\n self.pyoutput(new_feature)\n```\n\nIn this updated code, instead of setting `key` as attribute name, it sets \"Attribute\" as attribute name and `key` as its value. The actual `'Value'` field holds the corresponding value for that attribute or library.\n\nFor the \"Attributes to Expose\" field in the transformer parameters, you should now enter `Attribute, Value`. The output should be now a 2-column dataset with 'Attribute' and 'Value' columns. The attribute names will now be the row values in the `'Attribute'` column, and their associated values will be the row values in the `'Value'` column. This way we achieve a 'long format' data style, which should make the output easier to interpret and organize.\n",
"role": "assistant"
},
{
"content": "Ah, success. Now it works.",
"role": "user"
},
{
"content": "Great to hear that it's working now!\n\nJust for the sake of summarization, what was done here is we gathered various pieces of information about your Python environment \u2013 including the Python version, the location of Python executable, the site-packages path, and the version numbers of installed libraries. This information was then written into individual features on an attribute level.\n\nIf you have any more questions about FME, PythonCaller or anything else, feel free to ask. Happy data processing with FME!",
"role": "assistant"
}
],
"model": {
"id": "gpt-4",
"name": "GPT-4"
},
"name": "FME: report python environment details",
"prompt": "You are a large language model with a special focus on Feature Manipulation Engine (FME) and adept and helping users generate code for use in it's Regex, SQL, and Python editors.",
"temperature": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment