update MP job title script with new data format

Also uncomment the positions data on area page Fixes #592
mysociety · Jul 30, 2024 · e865ea6 · e865ea6
1 parent b69d60c
commit e865ea6
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 43 deletions.
diff --git a/hub/management/commands/import_mp_job_titles.py b/hub/management/commands/import_mp_job_titles.py
@@ -6,23 +6,12 @@
 import pandas as pd
 from tqdm import tqdm
 
-from hub.models import Area, AreaType, DataSet, DataType, Person, PersonData
-
-CONSTITUENCY_CORRECTIONS_DICT = {
-    "Beverly and Holderness": "Beverley and Holderness",
-    "Brighton Pavilion": "Brighton, Pavilion",
-    "Enfield Southgate": "Enfield, Southgate",
-    "Lewisham Deptford": "Lewisham, Deptford",
-    "Ealing Southall": "Ealing, Southall",
-    "Brighton Kemptown": "Brighton, Kemptown",
-    "Richmond": "Richmond (Yorks)",
-    "Na h-Eileanan an Lar": "Na h-Eileanan an Iar",
-}
+from hub.models import AreaType, DataSet, DataType, Person, PersonData
 
 
 class Command(BaseCommand):
     help = "Import MP Job titles"
-    data_file = settings.BASE_DIR / "data" / "mp_job_titles.csv"
+    data_file = settings.BASE_DIR / "data" / "mp_positions_jul_2024.csv"
 
     def handle(self, quiet=False, *args, **options):
         self._quiet = quiet
@@ -38,18 +27,8 @@ def get_df(self):
             return None
 
         df = pd.read_csv(
-            "data/mp_job_titles.csv",
-            usecols=["Constituency", "Short Title"],
-        )
-        df.columns = ["Title", "Constituency"]
-        df = df.query(
-            "not(Title.str.contains('^ ?(?:(?:MP|Member of Parliament) for .*|Member of .*Select Committee|Backbencher|Sadly died).*$'))"
-        ).copy()
-
-        # Clean the constituency data:
-        df.Constituency = df.Constituency.str.strip()
-        df.Constituency = df.Constituency.apply(
-            lambda x: CONSTITUENCY_CORRECTIONS_DICT.get(x, x)
+            self.data_file,
+            usecols=["mp", "position"],
         )
         return df
 
@@ -59,10 +38,10 @@ def create_data_type(self):
             defaults={
                 "data_type": "text",
                 "description": "Positions such as cabinet and shadow minister roles, spokespeople, and whips.",
-                "release_date": "January 2024",
+                "release_date": "July 2024",
                 "label": "MP positions (job titles)",
-                "source_label": "Data from Green Alliance.",
-                "source": "https://green-alliance.org.uk/",
+                "source_label": "Data compiled by mySociety from Gov.uk and Parliament.",
+                "source": "https://www.mysociety.org",
                 "table": "people__persondata",
                 "comparators": DataSet.string_comparators(),
             },
@@ -79,19 +58,17 @@ def create_data_type(self):
         return mp_job_titles
 
     def get_results(self, df: pd.DataFrame):
-        mps = Person.objects.filter(person_type="MP")
         results = {}
         if not self._quiet:
             print("Matching MPs with titles")
-        area_type = self.get_area_type()
         for index, row in df.iterrows():
+            if pd.isna(row.mp):
+                continue
             try:
-                area = Area.objects.get(
-                    name__iexact=row.Constituency, area_type=area_type
-                )
-                results[mps.get(area=area)] = row.Title
-            except Area.DoesNotExist:
-                print(f"Constituency: {row.Constituency} not found.")
+                person = Person.objects.get(name=row.mp, person_type="MP")
+                results[person] = row.position
+            except Person.DoesNotExist:
+                print(f"MP: {row.mp} not found.")
         return results
 
     def add_results(self, results, data_type):

diff --git a/hub/templates/hub/area.html b/hub/templates/hub/area.html
@@ -223,11 +223,6 @@ <h3 class="h5">MP profiles</h3>
                     <div class="card-header">
                         <h3 class="h5">Positions and memberships</h3>
                     </div>
-                    <div class="card-body text-muted">
-                        <p>We’re currently working on collecting updated data on MP positions and memberships.</p>
-                        <p>You can <a href="#signup">sign up to be notified when we add it</a>.</p>
-                    </div>
-                  {% comment %}
                     <div class="card-body">
                       {% if mp.job_titles %}
                         <h4 class="visually-hidden">Positions</h4>
@@ -270,10 +265,13 @@ <h4 class="h6 text-muted fw-bold">APPGs</h4>
                         </ul>
                       {% endif %}
                     </div>
+                    <div class="card-body text-muted">
+                        <p>We’re currently working on collecting updated data on MP memberships.</p>
+                        <p>You can <a href="#signup">sign up to be notified when we add it</a>.</p>
+                    </div>
                     <div class="card-footer">
-                        <p class="card-text">Data from <a href="https://www.parliament.uk/about/mps-and-lords/members/">UK Parliament</a>{% if mp.job_titles %} and <a href="https://green-alliance.org.uk/">Green Alliance</a>{% endif %}.</p>
+                        <p class="card-text">Data from <a href="https://www.parliament.uk/about/mps-and-lords/members/">UK Parliament</a>{% if mp.job_titles %} and <a href="http://www.mysociety.org">mySociety</a>{% endif %}.</p>
                     </div>
-                  {% endcomment %}
                 </div>
 
                 <div class="card dataset-card area-data--md area-data--featured">