Skip to content

Commit 4170b9e

Browse files
jtojnarjtojnar
committed
Use generators for database access
This will make it possible to GC memory when working with huge number of items.
1 parent 07992bc commit 4170b9e

File tree

9 files changed

+32
-35
lines changed

9 files changed

+32
-35
lines changed

‎src/daos/Items.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ public function bulkStatusUpdate(array $statuses): void {
155155
$this->backend->bulkStatusUpdate($statuses);
156156
}
157157

158-
public function getRaw(): array {
158+
public function getRaw(): iterable {
159159
return $this->backend->getRaw();
160160
}
161161

‎src/daos/ItemsInterface.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,9 @@ public function bulkStatusUpdate(array $statuses): void;
187187
/**
188188
* returns raw items table contents
189189
*
190-
* @return array<array{author: string, content: string, datetime: string, icon: string, id: int, lastseen: string, link: string, shared: int, source: int, starred: bool, thumbnail: ?string, title: string, uid: string, unread: bool, updatetime: string}> of all items
190+
* @return iterable<int, array{author: string, content: string, datetime: string, icon: string, id: int, lastseen: string, link: string, shared: int, source: int, starred: bool, thumbnail: ?string, title: string, uid: string, unread: bool, updatetime: string}> of all items
191191
*/
192-
public function getRaw(): array;
192+
public function getRaw(): iterable;
193193

194194
/**
195195
* inserts raw data into items table

‎src/daos/Sources.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,8 @@ public function validate(string $title, string $spout, array $params) {
197197
return true;
198198
}
199199

200-
public function getRaw(): array {
201-
/** @var array<array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> */
200+
public function getRaw(): iterable {
201+
/** @var iterable<array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> */
202202
$sources = $this->backend->getRaw();
203203

204204
return $sources;

‎src/daos/SourcesInterface.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,9 @@ public function checkIfExists(string $title, string $spout, array $params): int;
114114
/**
115115
* returns raw sources table contents
116116
*
117-
* @return array<array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> of all sources
117+
* @return iterable<int, array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> of all sources
118118
*/
119-
public function getRaw(): array;
119+
public function getRaw(): iterable;
120120

121121
/**
122122
* inserts raw data into sources table

‎src/daos/StatementsInterface.php‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -100,16 +100,15 @@ public static function bool(bool $bool): string;
100100
public static function datetime(\DateTime $date): string;
101101

102102
/**
103-
* Ensure row values have the appropriate PHP type. This assumes we are
104-
* using buffered queries (sql results are in PHP memory);.
103+
* Ensure row values have the appropriate PHP type.
105104
*
106-
* @param array<array<mixed>> $rows array of associative array representing row results
105+
* @param iterable<array<mixed>> $rows array of associative array representing row results
107106
* @param array<string, DatabaseInterface::PARAM_*> $expectedRowTypes associative array mapping columns to PDO types
108107
*
109-
* @return array<array<mixed>> of associative array representing row results having
108+
* @return iterable<int, array<mixed>> of associative array representing row results having
110109
* expected types
111110
*/
112-
public static function ensureRowTypes(array $rows, array $expectedRowTypes): array;
111+
public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): iterable;
113112

114113
/**
115114
* convert string array to string for storage in table row

‎src/daos/mysql/Items.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ public function sync(int $sinceId, DateTime $notBefore, DateTime $since, int $ho
418418
* @return int lowest id of interest
419419
*/
420420
public function lowestIdOfInterest(): int {
421-
$lowest = static::$stmt::ensureRowTypes(
421+
$lowests = static::$stmt::ensureRowTypes(
422422
$this->database->exec(
423423
'SELECT id FROM ' . $this->configuration->dbPrefix . 'items AS items
424424
WHERE ' . static::$stmt::isTrue('unread') .
@@ -427,8 +427,8 @@ public function lowestIdOfInterest(): int {
427427
),
428428
['id' => DatabaseInterface::PARAM_INT]
429429
);
430-
if ($lowest) {
431-
return $lowest[0]['id'];
430+
foreach ($lowests as $lowest) {
431+
return $lowest['id'];
432432
}
433433

434434
return 0;
@@ -687,7 +687,7 @@ public function bulkStatusUpdate(array $statuses): void {
687687
}
688688
}
689689

690-
public function getRaw(): array {
690+
public function getRaw(): iterable {
691691
$stmt = static::$stmt;
692692
$items = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'items');
693693

‎src/daos/mysql/Sources.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ public function checkIfExists(string $title, string $spout, array $params): int
304304
return 0;
305305
}
306306

307-
public function getRaw(): array {
307+
public function getRaw(): iterable {
308308
$stmt = static::$stmt;
309309
$sources = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'sources');
310310

‎src/daos/mysql/Statements.php‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -138,17 +138,16 @@ public static function datetime(\DateTime $date): string {
138138
}
139139

140140
/**
141-
* Ensure row values have the appropriate PHP type. This assumes we are
142-
* using buffered queries (sql results are in PHP memory).
141+
* Ensure row values have the appropriate PHP type.
143142
*
144-
* @param array<array<mixed>> $rows array of associative array representing row results
143+
* @param iterable<array<mixed>> $rows array of associative array representing row results
145144
* @param array<string, DatabaseInterface::PARAM_*> $expectedRowTypes associative array mapping columns to PDO types
146145
*
147-
* @return array<array<mixed>> of associative array representing row results having
146+
* @return iterable<int, array<mixed>> of associative array representing row results having
148147
* expected types
149148
*/
150-
public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
151-
foreach ($rows as $rowIndex => $row) {
149+
public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): iterable {
150+
foreach ($rows as $row) {
152151
foreach ($expectedRowTypes as $columnIndex => $type) {
153152
if (array_key_exists($columnIndex, $row)) {
154153
if ($type & DatabaseInterface::PARAM_NULL) {
@@ -185,13 +184,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
185184
$value = null;
186185
}
187186
if ($value !== null) {
188-
$rows[$rowIndex][$columnIndex] = $value;
187+
$row[$columnIndex] = $value;
189188
}
190189
}
191190
}
192-
}
193191

194-
return $rows;
192+
yield $row;
193+
}
195194
}
196195

197196
/**

‎src/daos/pgsql/Statements.php‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,17 +74,16 @@ public static function csvRowMatches(string $column, string $value): string {
7474
}
7575

7676
/**
77-
* Ensure row values have the appropriate PHP type. This assumes we are
78-
* using buffered queries (sql results are in PHP memory).
77+
* Ensure row values have the appropriate PHP type.
7978
*
80-
* @param array<array<mixed>> $rows array of associative array representing row results
79+
* @param iterable<array<mixed>> $rows array of associative array representing row results
8180
* @param array<string, DatabaseInterface::PARAM_*> $expectedRowTypes associative array mapping columns to PDO types
8281
*
83-
* @return array<array<mixed>> of associative array representing row results having
82+
* @return iterable<int, array<mixed>> of associative array representing row results having
8483
* expected types
8584
*/
86-
public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
87-
foreach ($rows as $rowIndex => $row) {
85+
public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): iterable {
86+
foreach ($rows as $row) {
8887
foreach ($expectedRowTypes as $columnIndex => $type) {
8988
if (array_key_exists($columnIndex, $row)) {
9089
if ($type & DatabaseInterface::PARAM_NULL) {
@@ -110,13 +109,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
110109
$value = null;
111110
}
112111
if ($value !== null) {
113-
$rows[$rowIndex][$columnIndex] = $value;
112+
$row[$columnIndex] = $value;
114113
}
115114
}
116115
}
117-
}
118116

119-
return $rows;
117+
yield $row;
118+
}
120119
}
121120

122121
/**

0 commit comments

Comments
 (0)